Record memory usage after garbage collection (#6963)

Per conversation in #6362.
open-telemetry · Nov 8, 2022 · 177d9cd · 177d9cd
1 parent 2e66e20
commit 177d9cd
Show file tree

Hide file tree

Showing 4 changed files with 86 additions and 10 deletions.
diff --git a/...roovy/io/opentelemetry/instrumentation/javaagent/runtimemetrics/RuntimeMetricsTest.groovy b/...roovy/io/opentelemetry/instrumentation/javaagent/runtimemetrics/RuntimeMetricsTest.groovy
@@ -28,6 +28,7 @@ class RuntimeMetricsTest extends AgentInstrumentationSpecification {
       assert getMetrics().any { it.name == "process.runtime.jvm.memory.usage" }
       assert getMetrics().any { it.name == "process.runtime.jvm.memory.committed" }
       assert getMetrics().any { it.name == "process.runtime.jvm.memory.limit" }
+      assert getMetrics().any { it.name == "process.runtime.jvm.memory.usage_after_last_gc" }
       assert getMetrics().any { it.name == "process.runtime.jvm.threads.count" }
       assert getMetrics().any { it.name == "process.runtime.jvm.buffer.limit" }
       assert getMetrics().any { it.name == "process.runtime.jvm.buffer.count" }

diff --git a/instrumentation/runtime-metrics/library/README.md b/instrumentation/runtime-metrics/library/README.md
@@ -53,28 +53,34 @@ The following lists attributes reported for a variety of garbage collectors. Not
   * `process.runtime.jvm.memory.usage`: {pool=Compressed Class Space,type=non_heap}, {pool=Par Eden Space,type=heap}, {pool=Tenured Gen,type=heap}, {pool=Par Survivor Space,type=heap}, {pool=Code Cache,type=non_heap}, {pool=Metaspace,type=non_heap}
   * `process.runtime.jvm.memory.committed`: {pool=Compressed Class Space,type=non_heap}, {pool=Par Eden Space,type=heap}, {pool=Tenured Gen,type=heap}, {pool=Par Survivor Space,type=heap}, {pool=Code Cache,type=non_heap}, {pool=Metaspace,type=non_heap}
   * `process.runtime.jvm.memory.limit`: {pool=Compressed Class Space,type=non_heap}, {pool=Par Eden Space,type=heap}, {pool=Tenured Gen,type=heap}, {pool=Par Survivor Space,type=heap}, {pool=Code Cache,type=non_heap}
+  * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=Par Eden Space,type=heap}, {pool=Tenured Gen,type=heap}, {pool=Par Survivor Space,type=heap}
 * G1 Garbage Collector
   * `process.runtime.jvm.memory.init`: {pool=G1 Survivor Space,type=heap}, {pool=G1 Eden Space,type=heap}, {pool=CodeCache,type=non_heap}, {pool=G1 Old Gen,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
   * `process.runtime.jvm.memory.usage`: {pool=G1 Survivor Space,type=heap}, {pool=G1 Eden Space,type=heap}, {pool=CodeCache,type=non_heap}, {pool=G1 Old Gen,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
   * `process.runtime.jvm.memory.committed`: {pool=G1 Survivor Space,type=heap}, {pool=G1 Eden Space,type=heap}, {pool=CodeCache,type=non_heap}, {pool=G1 Old Gen,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
   * `process.runtime.jvm.memory.limit`: {pool=CodeCache,type=non_heap}, {pool=G1 Old Gen,type=heap}, {pool=Compressed Class Space,type=non_heap}
+  * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=G1 Survivor Space,type=heap}, {pool=G1 Eden Space,type=heap}, {pool=G1 Old Gen,type=heap}
 * Parallel Garbage Collector
   * `process.runtime.jvm.memory.init`: {pool=CodeCache,type=non_heap}, {pool=PS Survivor Space,type=heap}, {pool=PS Old Gen,type=heap}, {pool=PS Eden Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
   * `process.runtime.jvm.memory.usage`: {pool=CodeCache,type=non_heap}, {pool=PS Survivor Space,type=heap}, {pool=PS Old Gen,type=heap}, {pool=PS Eden Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
   * `process.runtime.jvm.memory.committed`: {pool=CodeCache,type=non_heap}, {pool=PS Survivor Space,type=heap}, {pool=PS Old Gen,type=heap}, {pool=PS Eden Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
   * `process.runtime.jvm.memory.limit`: {pool=CodeCache,type=non_heap}, {pool=PS Survivor Space,type=heap}, {pool=PS Old Gen,type=heap}, {pool=PS Eden Space,type=heap}, {pool=Compressed Class Space,type=non_heap}
+  * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=PS Survivor Space,type=heap}, {pool=PS Old Gen,type=heap}, {pool=PS Eden Space,type=heap}
 * Serial Garbage Collector
   * `process.runtime.jvm.memory.init`: {pool=CodeCache,type=non_heap}, {pool=Tenured Gen,type=heap}, {pool=Eden Space,type=heap}, {pool=Survivor Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
   * `process.runtime.jvm.memory.usage`: {pool=CodeCache,type=non_heap}, {pool=Tenured Gen,type=heap}, {pool=Eden Space,type=heap}, {pool=Survivor Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
   * `process.runtime.jvm.memory.committed`: {pool=CodeCache,type=non_heap}, {pool=Tenured Gen,type=heap}, {pool=Eden Space,type=heap}, {pool=Survivor Space,type=heap}, {pool=Compressed Class Space,type=non_heap}, {pool=Metaspace,type=non_heap}
   * `process.runtime.jvm.memory.limit`: {pool=CodeCache,type=non_heap}, {pool=Tenured Gen,type=heap}, {pool=Eden Space,type=heap}, {pool=Survivor Space,type=heap}, {pool=Compressed Class Space,type=non_heap}
+  * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=Tenured Gen,type=heap}, {pool=Eden Space,type=heap}, {pool=Survivor Space,type=heap}
 * Shenandoah Garbage Collector
   * `process.runtime.jvm.memory.init`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=Shenandoah,type=heap}, {pool=Compressed Class Space,type=non_heap}
   * `process.runtime.jvm.memory.usage`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=Shenandoah,type=heap}, {pool=Compressed Class Space,type=non_heap}
   * `process.runtime.jvm.memory.committed`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=Shenandoah,type=heap}, {pool=Compressed Class Space,type=non_heap}
   * `process.runtime.jvm.memory.limit`: {pool=CodeCache,type=non_heap}, {pool=Shenandoah,type=heap}, {pool=Compressed Class Space,type=non_heap}
+  * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=Shenandoah,type=heap}
 * Z Garbage Collector
   * `process.runtime.jvm.memory.init`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=ZHeap,type=heap}, {pool=Compressed Class Space,type=non_heap}
   * `process.runtime.jvm.memory.usage`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=ZHeap,type=heap}, {pool=Compressed Class Space,type=non_heap}
   * `process.runtime.jvm.memory.committed`: {pool=Metaspace,type=non_heap}, {pool=CodeCache,type=non_heap}, {pool=ZHeap,type=heap}, {pool=Compressed Class Space,type=non_heap}
   * `process.runtime.jvm.memory.limit`: {pool=CodeCache,type=non_heap}, {pool=ZHeap,type=heap}, {pool=Compressed Class Space,type=non_heap}
+  * `process.runtime.jvm.memory.usage_after_last_gc`: {pool=ZHeap,type=heap}
diff --git a/...cs/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java b/...cs/library/src/main/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPools.java
@@ -34,7 +34,8 @@
  *   process.runtime.jvm.memory.init{type="heap",pool="G1 Eden Space"} 1000000
  *   process.runtime.jvm.memory.usage{type="heap",pool="G1 Eden Space"} 2500000
  *   process.runtime.jvm.memory.committed{type="heap",pool="G1 Eden Space"} 3000000
- *   process.runtime.jvm.memory.max{type="heap",pool="G1 Eden Space"} 4000000
+ *   process.runtime.jvm.memory.limit{type="heap",pool="G1 Eden Space"} 4000000
+ *   process.runtime.jvm.memory.usage_after_last_gc{type="heap",pool="G1 Eden Space"} 1500000
  *   process.runtime.jvm.memory.init{type="non_heap",pool="Metaspace"} 200
  *   process.runtime.jvm.memory.usage{type="non_heap",pool="Metaspace"} 400
  *   process.runtime.jvm.memory.committed{type="non_heap",pool="Metaspace"} 500
@@ -61,30 +62,41 @@ static void registerObservers(OpenTelemetry openTelemetry, List<MemoryPoolMXBean
         .upDownCounterBuilder("process.runtime.jvm.memory.usage")
         .setDescription("Measure of memory used")
         .setUnit("By")
-        .buildWithCallback(callback(poolBeans, MemoryUsage::getUsed));
+        .buildWithCallback(callback(poolBeans, MemoryPoolMXBean::getUsage, MemoryUsage::getUsed));
 
     meter
         .upDownCounterBuilder("process.runtime.jvm.memory.init")
         .setDescription("Measure of initial memory requested")
         .setUnit("By")
-        .buildWithCallback(callback(poolBeans, MemoryUsage::getInit));
+        .buildWithCallback(callback(poolBeans, MemoryPoolMXBean::getUsage, MemoryUsage::getInit));
 
     meter
         .upDownCounterBuilder("process.runtime.jvm.memory.committed")
         .setDescription("Measure of memory committed")
         .setUnit("By")
-        .buildWithCallback(callback(poolBeans, MemoryUsage::getCommitted));
+        .buildWithCallback(
+            callback(poolBeans, MemoryPoolMXBean::getUsage, MemoryUsage::getCommitted));
 
     meter
         .upDownCounterBuilder("process.runtime.jvm.memory.limit")
         .setDescription("Measure of max obtainable memory")
         .setUnit("By")
-        .buildWithCallback(callback(poolBeans, MemoryUsage::getMax));
+        .buildWithCallback(callback(poolBeans, MemoryPoolMXBean::getUsage, MemoryUsage::getMax));
+
+    meter
+        .upDownCounterBuilder("process.runtime.jvm.memory.usage_after_last_gc")
+        .setDescription(
+            "Measure of memory used after the most recent garbage collection event on this pool")
+        .setUnit("By")
+        .buildWithCallback(
+            callback(poolBeans, MemoryPoolMXBean::getCollectionUsage, MemoryUsage::getUsed));
   }
 
   // Visible for testing
   static Consumer<ObservableLongMeasurement> callback(
-      List<MemoryPoolMXBean> poolBeans, Function<MemoryUsage, Long> extractor) {
+      List<MemoryPoolMXBean> poolBeans,
+      Function<MemoryPoolMXBean, MemoryUsage> memoryUsageExtractor,
+      Function<MemoryUsage, Long> valueExtractor) {
     List<Attributes> attributeSets = new ArrayList<>(poolBeans.size());
     for (MemoryPoolMXBean pool : poolBeans) {
       attributeSets.add(
@@ -97,7 +109,13 @@ static Consumer<ObservableLongMeasurement> callback(
     return measurement -> {
       for (int i = 0; i < poolBeans.size(); i++) {
         Attributes attributes = attributeSets.get(i);
-        long value = extractor.apply(poolBeans.get(i).getUsage());
+        MemoryUsage memoryUsage = memoryUsageExtractor.apply(poolBeans.get(i));
+        if (memoryUsage == null) {
+          // JVM may return null in special cases for MemoryPoolMXBean.getUsage() and
+          // MemoryPoolMXBean.getCollectionUsage()
+          continue;
+        }
+        long value = valueExtractor.apply(memoryUsage);
         if (value != -1) {
           measurement.record(value, attributes);
         }

diff --git a/...ibrary/src/test/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPoolsTest.java b/...ibrary/src/test/java/io/opentelemetry/instrumentation/runtimemetrics/MemoryPoolsTest.java
@@ -8,6 +8,7 @@
 import static io.opentelemetry.instrumentation.runtimemetrics.ScopeUtil.EXPECTED_SCOPE;
 import static io.opentelemetry.sdk.testing.assertj.OpenTelemetryAssertions.assertThat;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
@@ -22,6 +23,7 @@
 import java.lang.management.MemoryType;
 import java.lang.management.MemoryUsage;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.function.Consumer;
 import org.junit.jupiter.api.BeforeEach;
@@ -31,8 +33,11 @@
 import org.mockito.Mock;
 import org.mockito.Spy;
 import org.mockito.junit.jupiter.MockitoExtension;
+import org.mockito.junit.jupiter.MockitoSettings;
+import org.mockito.quality.Strictness;
 
 @ExtendWith(MockitoExtension.class)
+@MockitoSettings(strictness = Strictness.LENIENT)
 class MemoryPoolsTest {
 
   @RegisterExtension
@@ -45,6 +50,8 @@ class MemoryPoolsTest {
 
   @Mock private MemoryUsage heapPoolUsage;
   @Mock private MemoryUsage nonHeapUsage;
+  @Mock private MemoryUsage heapCollectionUsage;
+  @Mock private MemoryUsage nonHeapCollectionUsage;
 
   private List<MemoryPoolMXBean> beans;
 
@@ -53,9 +60,11 @@ void setup() {
     when(heapPoolBean.getName()).thenReturn("heap_pool");
     when(heapPoolBean.getType()).thenReturn(MemoryType.HEAP);
     when(heapPoolBean.getUsage()).thenReturn(heapPoolUsage);
+    when(heapPoolBean.getCollectionUsage()).thenReturn(heapCollectionUsage);
     when(nonHeapPoolBean.getName()).thenReturn("non_heap_pool");
     when(nonHeapPoolBean.getType()).thenReturn(MemoryType.NON_HEAP);
     when(nonHeapPoolBean.getUsage()).thenReturn(nonHeapUsage);
+    when(nonHeapPoolBean.getCollectionUsage()).thenReturn(nonHeapCollectionUsage);
     beans = Arrays.asList(heapPoolBean, nonHeapPoolBean);
   }
 
@@ -69,7 +78,8 @@ void registerObservers() {
     when(nonHeapUsage.getUsed()).thenReturn(15L);
     when(nonHeapUsage.getCommitted()).thenReturn(16L);
     when(nonHeapUsage.getMax()).thenReturn(17L);
-
+    when(heapCollectionUsage.getUsed()).thenReturn(18L);
+    when(nonHeapCollectionUsage.getUsed()).thenReturn(19L);
     MemoryPools.registerObservers(testing.getOpenTelemetry(), beans);
 
     testing.waitAndAssertMetrics(
@@ -176,6 +186,33 @@ void registerObservers() {
                                                 AttributeKey.stringKey("pool"), "non_heap_pool")
                                             .hasAttribute(
                                                 AttributeKey.stringKey("type"), "non_heap")))));
+    testing.waitAndAssertMetrics(
+        "io.opentelemetry.runtime-metrics",
+        "process.runtime.jvm.memory.usage_after_last_gc",
+        metrics ->
+            metrics.anySatisfy(
+                metricData ->
+                    assertThat(metricData)
+                        .hasInstrumentationScope(EXPECTED_SCOPE)
+                        .hasDescription(
+                            "Measure of memory used after the most recent garbage collection event on this pool")
+                        .hasUnit("By")
+                        .hasLongSumSatisfying(
+                            sum ->
+                                sum.hasPointsSatisfying(
+                                    point ->
+                                        point
+                                            .hasValue(18)
+                                            .hasAttribute(
+                                                AttributeKey.stringKey("pool"), "heap_pool")
+                                            .hasAttribute(AttributeKey.stringKey("type"), "heap"),
+                                    point ->
+                                        point
+                                            .hasValue(19)
+                                            .hasAttribute(
+                                                AttributeKey.stringKey("pool"), "non_heap_pool")
+                                            .hasAttribute(
+                                                AttributeKey.stringKey("type"), "non_heap")))));
   }
 
   @Test
@@ -184,7 +221,7 @@ void callback_Records() {
     when(nonHeapUsage.getUsed()).thenReturn(2L);
 
     Consumer<ObservableLongMeasurement> callback =
-        MemoryPools.callback(beans, MemoryUsage::getUsed);
+        MemoryPools.callback(beans, MemoryPoolMXBean::getUsage, MemoryUsage::getUsed);
     callback.accept(measurement);
 
     verify(measurement)
@@ -199,11 +236,25 @@ void callback_SkipRecord() {
     when(heapPoolUsage.getMax()).thenReturn(1L);
     when(nonHeapUsage.getMax()).thenReturn(-1L);
 
-    Consumer<ObservableLongMeasurement> callback = MemoryPools.callback(beans, MemoryUsage::getMax);
+    Consumer<ObservableLongMeasurement> callback =
+        MemoryPools.callback(beans, MemoryPoolMXBean::getUsage, MemoryUsage::getMax);
     callback.accept(measurement);
 
     verify(measurement)
         .record(1, Attributes.builder().put("pool", "heap_pool").put("type", "heap").build());
     verify(measurement, never()).record(eq(-1), any());
   }
+
+  @Test
+  void callback_NullUsage() {
+    when(heapPoolBean.getCollectionUsage()).thenReturn(null);
+
+    Consumer<ObservableLongMeasurement> callback =
+        MemoryPools.callback(
+            Collections.singletonList(heapPoolBean),
+            MemoryPoolMXBean::getCollectionUsage,
+            MemoryUsage::getUsed);
+    callback.accept(measurement);
+    verify(measurement, never()).record(anyLong(), any());
+  }
 }