introduce info bpstats metrics

microsoft · Sep 12, 2024 · 9a6bca1 · 9a6bca1
1 parent a9a7f72
commit 9a6bca1
Show file tree

Hide file tree

Showing 10 changed files with 107 additions and 27 deletions.
diff --git a/libs/cluster/Server/ClusterProvider.cs b/libs/cluster/Server/ClusterProvider.cs
@@ -271,6 +271,14 @@ public MetricsItem[] GetGossipStats(bool metricsDisabled)
                 ];
         }
 
+        public MetricsItem[] GetBufferPoolStats()
+        {
+            return [
+                new("migration_manager", migrationManager.GetBufferPoolStats()),
+                new("replication_manager", replicationManager.GetBufferPoolStats())
+                ];
+        }
+
         internal ReplicationLogCheckpointManager GetReplicationLogCheckpointManager(StoreType storeType)
         {
             Debug.Assert(serverOptions.EnableCluster);

diff --git a/libs/cluster/Server/Migration/MigrationManager.cs b/libs/cluster/Server/Migration/MigrationManager.cs
@@ -54,6 +54,8 @@ public bool Purge()
             return true;
         }
 
+        public string GetBufferPoolStats() => networkBuffers.GetStats();
+
         /// <summary>
         /// Get number of active migrate sessions
         /// </summary>

diff --git a/libs/cluster/Server/Replication/ReplicationManager.cs b/libs/cluster/Server/Replication/ReplicationManager.cs
@@ -145,6 +145,8 @@ public bool Purge()
             return true;
         }
 
+        public string GetBufferPoolStats() => networkBuffers.GetStats();
+
         void CheckpointVersionShift(bool isMainStore, long oldVersion, long newVersion)
         {
             if (clusterProvider.clusterManager.CurrentConfig.LocalNodeRole == NodeRole.REPLICA)

diff --git a/libs/common/Format.cs b/libs/common/Format.cs
@@ -168,6 +168,10 @@ public static string GetHostName(ILogger? logger = null)
 
             return "";
         }
+
+        public static string MegaBytes(long size) => (((size - 1) >> 20) + 1).ToString();
+
+        public static string KiloBytes(long size) =>(((size - 1) >> 10) + 1).ToString();
     }
 #pragma warning restore format
 }
diff --git a/libs/common/Memory/LimitedFixedBufferPool.cs b/libs/common/Memory/LimitedFixedBufferPool.cs
@@ -147,34 +147,57 @@ public void Dispose()
         }
 
         /// <summary>
-        /// Print pool contents
+        /// Get statistics for this buffer pool
         /// </summary>
-        public void Print()
+        /// <returns></returns>
+        public string GetStats()
         {
-            for (int i = 0; i < numLevels; i++)
+            var stats = $"totalAllocations = {totalAllocations}, " +
+                $"numLevels = {numLevels}, " +
+                $"maxEntriesPerLevel = {maxEntriesPerLevel}";
+
+            var bufferStats = "";
+            var totalBufferCount = 0;
+            for (var i = 0; i < numLevels; i++)
             {
                 if (pool[i] == null) continue;
-                foreach (var item in pool[i].items)
+
+                var count = pool[i].items.Count;
+
+                if (count == 0) continue;
+
+                totalBufferCount += count;
+                bufferStats += $"<{count},{Format.KiloBytes(minAllocationSize * (i + 1))}KB>";
+
+                // Keep trying Dequeuing until no items left to free
+                while (pool[i].items.TryDequeue(out var entry))
                 {
-                    Console.WriteLine("  " + item.entry.Length.ToString());
+                    entry = null;
+                    Interlocked.Decrement(ref pool[i].size);
                 }
             }
+
+            if (totalBufferCount > 0)
+                stats += $", totalBufferCount:{totalBufferCount}[" + bufferStats + "]";
+
+            return stats;
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         int Position(int v)
         {
             if (v < minAllocationSize || !BitOperations.IsPow2(v))
                 return -1;
+            return GetLevel(minAllocationSize, v);
+        }
 
-            v /= minAllocationSize;
-
-            if (v == 1) return 0;
+        public static int GetLevel(int minAllocationSize, int requestedSize)
+        {
+            Debug.Assert(BitOperations.IsPow2(minAllocationSize));
+            Debug.Assert(BitOperations.IsPow2(requestedSize));
+            requestedSize /= minAllocationSize;
 
-            int level = BitOperations.Log2((uint)v - 1) + 1;
-            if (level >= numLevels)
-                return -1;
-            return level;
+            return requestedSize == 1 ? 0 : BitOperations.Log2((uint)requestedSize - 1) + 1;
         }
     }
 }
diff --git a/libs/common/Metrics/InfoMetricsType.cs b/libs/common/Metrics/InfoMetricsType.cs
@@ -72,6 +72,10 @@ public enum InfoMetricsType : byte
         /// Modules info
         /// </summary>
         MODULES,
+        /// <summary>
+        /// Shared buffer pool stats
+        /// </summary>
+        BPSTATS,
     }
 
     /// <summary>

diff --git a/libs/common/NetworkBuffers.cs b/libs/common/NetworkBuffers.cs
@@ -61,10 +61,10 @@ public NetworkBuffers Allocate(int maxEntriesPerLevel = 16, ILogger logger = nul
             var minSize = Math.Min(sendMinAllocationSize, recvMinAllocationSize);
             var maxSize = Math.Max(sendMinAllocationSize, recvMinAllocationSize);
 
-            var levels = (maxSize / minSize) + 1;
-            Debug.Assert(levels > 0);
+            var levels = LimitedFixedBufferPool.GetLevel(recvMinAllocationSize, sendMinAllocationSize) + 1;
+            Debug.Assert(levels >= 0);
             levels = Math.Max(4, levels);
-            bufferPool = new LimitedFixedBufferPool(sendMinAllocationSize, maxEntriesPerLevel: maxEntriesPerLevel, numLevels: levels, logger: logger);
+            bufferPool = new LimitedFixedBufferPool(recvMinAllocationSize, maxEntriesPerLevel: maxEntriesPerLevel, numLevels: levels, logger: logger);
             return this;
         }
 
@@ -77,6 +77,15 @@ public void Purge()
             bufferPool?.Purge();
         }
 
+        /// <summary>
+        /// Get buffer pool statistics
+        /// </summary>
+        /// <returns></returns>
+        public string GetStats()
+        {
+            return bufferPool.GetStats();
+        }
+
         /// <summary>
         /// Dispose associated network buffer pool
         /// </summary>

diff --git a/libs/server/Cluster/IClusterProvider.cs b/libs/server/Cluster/IClusterProvider.cs
@@ -43,6 +43,12 @@ public interface IClusterProvider : IDisposable
         /// </summary>
         MetricsItem[] GetReplicationInfo();
 
+        /// <summary>
+        /// Get buffer poolt stats
+        /// </summary>
+        /// <returns></returns>
+        MetricsItem[] GetBufferPoolStats();
+
         /// <summary>
         /// Is replica
         /// </summary>

diff --git a/libs/server/Metrics/Info/GarnetInfoMetrics.cs b/libs/server/Metrics/Info/GarnetInfoMetrics.cs
@@ -37,6 +37,7 @@ class GarnetInfoMetrics
         MetricsItem[] persistenceInfo = null;
         MetricsItem[] clientsInfo = null;
         MetricsItem[] keyspaceInfo = null;
+        MetricsItem[] bufferPoolStats = null;
 
         public GarnetInfoMetrics() { }
 
@@ -224,15 +225,15 @@ private void PopulateObjectStoreStats(StoreWrapper storeWrapper)
                 ];
         }
 
-        public void PopulateStoreHashDistribution(StoreWrapper storeWrapper) => storeHashDistrInfo = [new("", storeWrapper.store.DumpDistribution())];
+        private void PopulateStoreHashDistribution(StoreWrapper storeWrapper) => storeHashDistrInfo = [new("", storeWrapper.store.DumpDistribution())];
 
-        public void PopulateObjectStoreHashDistribution(StoreWrapper storeWrapper) => objectStoreHashDistrInfo = [new("", storeWrapper.objectStore.DumpDistribution())];
+        private void PopulateObjectStoreHashDistribution(StoreWrapper storeWrapper) => objectStoreHashDistrInfo = [new("", storeWrapper.objectStore.DumpDistribution())];
 
-        public void PopulateStoreRevivInfo(StoreWrapper storeWrapper) => storeRevivInfo = [new("", storeWrapper.store.DumpRevivificationStats())];
+        private void PopulateStoreRevivInfo(StoreWrapper storeWrapper) => storeRevivInfo = [new("", storeWrapper.store.DumpRevivificationStats())];
 
-        public void PopulateObjectStoreRevivInfo(StoreWrapper storeWrapper) => objectStoreRevivInfo = [new("", storeWrapper.objectStore.DumpRevivificationStats())];
+        private void PopulateObjectStoreRevivInfo(StoreWrapper storeWrapper) => objectStoreRevivInfo = [new("", storeWrapper.objectStore.DumpRevivificationStats())];
 
-        public void PopulatePersistenceInfo(StoreWrapper storeWrapper)
+        private void PopulatePersistenceInfo(StoreWrapper storeWrapper)
         {
             bool aofEnabled = storeWrapper.serverOptions.EnableAOF;
             persistenceInfo =
@@ -258,6 +259,14 @@ private void PopulateKeyspaceInfo(StoreWrapper storeWrapper)
             keyspaceInfo = null;
         }
 
+        private void PopulateClusterBufferPoolStats(StoreWrapper storeWrapper)
+        {
+            if (storeWrapper.clusterProvider != null)
+            {
+                bufferPoolStats = storeWrapper.clusterProvider.GetBufferPoolStats();
+            }
+        }
+
         public static string GetSectionHeader(InfoMetricsType infoType)
         {
             return infoType switch
@@ -277,6 +286,7 @@ public static string GetSectionHeader(InfoMetricsType infoType)
                 InfoMetricsType.CLIENTS => "Clients",
                 InfoMetricsType.KEYSPACE => "Keyspace",
                 InfoMetricsType.MODULES => "Modules",
+                InfoMetricsType.BPSTATS => "BufferPool Stats",
                 _ => "Default",
             };
         }
@@ -353,6 +363,9 @@ public string GetRespInfo(InfoMetricsType section, StoreWrapper storeWrapper)
                     return GetSectionRespInfo(InfoMetricsType.KEYSPACE, keyspaceInfo);
                 case InfoMetricsType.MODULES:
                     return GetSectionRespInfo(section, null);
+                case InfoMetricsType.BPSTATS:
+                    PopulateClusterBufferPoolStats(storeWrapper);
+                    return GetSectionRespInfo(InfoMetricsType.BPSTATS, bufferPoolStats);
                 default:
                     return "";
             }
@@ -364,7 +377,9 @@ public string GetRespInfo(InfoMetricsType[] sections, StoreWrapper storeWrapper)
             for (var i = 0; i < sections.Length; i++)
             {
                 var section = sections[i];
-                response += GetRespInfo(section, storeWrapper);
+                var resp = GetRespInfo(section, storeWrapper);
+                if (string.IsNullOrEmpty(resp)) continue;
+                response += resp;
                 response += sections.Length - 1 == i ? "" : "\r\n";
             }
             return response;

diff --git a/libs/server/Metrics/Info/InfoCommand.cs b/libs/server/Metrics/Info/InfoCommand.cs
@@ -3,7 +3,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using Garnet.common;
 
 namespace Garnet.server
@@ -70,11 +69,19 @@ private bool NetworkINFO()
             }
             else
             {
-                InfoMetricsType[] sectionsArr = sections == null ? GarnetInfoMetrics.defaultInfo : [.. sections];
-                GarnetInfoMetrics garnetInfo = new();
-                string info = garnetInfo.GetRespInfo(sectionsArr, storeWrapper);
-                while (!RespWriteUtils.WriteAsciiBulkString(info, ref dcurr, dend))
-                    SendAndReset();
+                var sectionsArr = sections == null ? GarnetInfoMetrics.defaultInfo : [.. sections];
+                var garnetInfo = new GarnetInfoMetrics();
+                var info = garnetInfo.GetRespInfo(sectionsArr, storeWrapper);
+                if (!string.IsNullOrEmpty(info))
+                {
+                    while (!RespWriteUtils.WriteAsciiBulkString(info, ref dcurr, dend))
+                        SendAndReset();
+                }
+                else
+                {
+                    while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_ERRNOTFOUND, ref dcurr, dend))
+                        SendAndReset();
+                }
             }
             return true;