PR 1: Detect undersized BlueStore DB devices

hotsos/core/host_helpers/cli/catalog.py

@@ -74,6 +74,14 @@ def __init__(self):
             'ceph_daemon_osd_perf_dump':
                 [BinCmd('ceph daemon osd.{osd_id} perf dump',
                         json_decode=True),
+                 FileCmd('sos_commands/ceph_osd/'
+                         'ceph_daemon_.var.run.ceph.'
+                         'ceph-osd.{osd_id}.asok_perf_dump',
+                         json_decode=True),
+                 FileCmd('sos_commands/ceph_osd/'
+                         'ceph_daemon_.var.snap.microceph.current.run.'
+                         'ceph-osd.{osd_id}.asok_perf_dump',
+                         json_decode=True),
                  # requires sosreport 4.3 or above
                  FileCmd('sos_commands/ceph_osd/'
                          'ceph_daemon_osd.{osd_id}_perf_dump',

hotsos/core/plugins/storage/ceph/common.py

@@ -179,6 +179,9 @@ class CephInstallInfo(InstallInfoBase):
 
 class CephChecks(StorageBase):
     """ Ceph Checks. """
+    # A 1 GiB BlueStore DB is the ceph-volume default and is generally too
+    # small for production OSDs.
+    BLUESTORE_DB_SIZE_MIN = 5 * 1024 * 1024 * 1024
     # Threshold above which an OSD's bluefs log is considered oversized.
     # Healthy OSDs keep this well under 50 GiB; sustained growth past this
     # point indicates that bluefs log compaction has failed and the log is
@@ -444,6 +447,29 @@ def local_osds_with_oversized_bluefs_log(self):
 
         return sorted(bad)
 
+    @cached_property
+    def local_osds_with_small_bluestore_db(self):
+        """Return local OSDs with a BlueStore DB device at most 5 GiB."""
+        bad = []
+        for osd in self.local_osds:
+            try:
+                bluefs = CephDaemonPerfDump(osd_id=osd.id).bluefs
+            except Exception:  # pylint: disable=broad-except
+                continue
+
+            try:
+                db_size = int(bluefs['db_total_bytes'])
+            except (KeyError, TypeError, ValueError):
+                continue
+
+            if db_size <= 0:
+                continue
+
+            if db_size <= self.BLUESTORE_DB_SIZE_MIN:
+                bad.append(f'osd.{osd.id}')
+
+        return sorted(set(bad))
+
     @cached_property
     def bluestore_enabled(self):
         """

hotsos/defs/scenarios/storage/ceph/ceph-osd/small_bluestore_db.yaml

@@ -0,0 +1,16 @@
+checks:
+  osds_with_small_bluestore_db:
+    property:
+      path: hotsos.core.plugins.storage.ceph.CephChecks.local_osds_with_small_bluestore_db
+      ops: [[length_hint], [gt, 0]]
+conclusions:
+  osds-have-small-bluestore-db:
+    decision: osds_with_small_bluestore_db
+    raises:
+      type: CephOSDWarning
+      message: >-
+        Found OSD(s) {bad_osds} with a BlueStore DB device of 5 GiB or smaller. A small
+        DB device can cause metadata to spill over to the main OSD device and degrade
+        performance. Review the OSD device layout and provision a larger fast DB device.
+      format-dict:
+        bad_osds: '@checks.osds_with_small_bluestore_db.requires.value_actual:comma_join'

hotsos/defs/tests/scenarios/storage/ceph/ceph-osd/small_bluestore_db.yaml

@@ -0,0 +1,11 @@
+mock:
+  patch:
+    hotsos.core.plugins.storage.ceph.CephChecks.local_osds_with_small_bluestore_db:
+      kwargs:
+        new: ['osd.0']
+raised-issues:
+  CephOSDWarning: >-
+    Found OSD(s) osd.0 with a BlueStore DB device of 5 GiB or smaller. A small
+    DB device can cause metadata to spill over to the main OSD device and
+    degrade performance. Review the OSD device layout and provision a larger
+    fast DB device.

tests/unit/storage/test_ceph_common.py

@@ -103,6 +103,27 @@ def test_ceph_dep_snap(self):
 
 class TestCephChecks(CephCommonTestsBase):
     """ Unit tests for ceph checks. """
+    @utils.create_data_root(
+        {'sos_commands/ceph_osd/'
+         'ceph_daemon_.var.run.ceph.ceph-osd.0.asok_perf_dump': (
+             '{"bluefs": {"db_total_bytes": 1073741824}}'),
+         'sos_commands/ceph_osd/'
+         'ceph_daemon_.var.run.ceph.ceph-osd.1.asok_perf_dump': (
+             '{"bluefs": {"db_total_bytes": 10737418240}}'),
+         'sos_commands/ceph_osd/ceph-volume_lvm_list': (
+             '====== osd.0 =======\n'
+             '  osd fsid                  test-osd-fsid-0\n'
+             '  devices                   /dev/mapper/ceph-osd-0\n'
+             '====== osd.1 =======\n'
+             '  osd fsid                  test-osd-fsid-1\n'
+             '  devices                   /dev/mapper/ceph-osd-1\n')})
+    def test_local_osds_with_small_bluestore_db(self):
+        """Test BlueStore DB devices at or below the minimum size are found."""
+        checks = ceph.common.CephChecks()
+
+        self.assertEqual(checks.local_osds_with_small_bluestore_db,
+                         ['osd.0'])
+
     def test_mds_balancer_disabled_by_interval(self):
         """Test mds balancer disabled by interval."""
         cases = [

tests/unit/storage/test_ceph_osd.py

@@ -121,6 +121,15 @@ def test_daemon_osd_perf_dump_no_exist(self):
         perf = ceph.common.CephDaemonPerfDump(osd_id=100)
         self.assertEqual(perf.bluefs, {})
 
+    @utils.create_data_root(
+        {'sos_commands/ceph_osd/'
+         'ceph_daemon_.var.snap.microceph.current.run.ceph-osd.0.'
+         'asok_perf_dump': '{"bluefs": {"db_total_bytes": 1073741824}}'})
+    def test_daemon_osd_perf_dump_microceph(self):
+        """Test MicroCeph OSD perf dump values are accessible."""
+        perf = ceph.common.CephDaemonPerfDump(osd_id=0)
+        self.assertEqual(perf.bluefs.get('db_total_bytes'), 1073741824)
+
     def test_oversized_bluefs_log_no_issue(self):
         """Test no oversized bluefs log with healthy data."""
         checks = ceph.common.CephChecks()