From 80721e2851f90e2b7d0d0fc92aae79f982314cb9 Mon Sep 17 00:00:00 2001 From: lastpeony Date: Fri, 25 Oct 2024 17:58:45 +0300 Subject: [PATCH 1/7] change available memory calculation if running inside a container --- src/main/java/io/antmedia/SystemUtils.java | 58 ++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/main/java/io/antmedia/SystemUtils.java b/src/main/java/io/antmedia/SystemUtils.java index f27dbb57c..6ad4e4cec 100644 --- a/src/main/java/io/antmedia/SystemUtils.java +++ b/src/main/java/io/antmedia/SystemUtils.java @@ -5,6 +5,10 @@ import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; import java.lang.reflect.Method; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; import javax.management.MBeanServer; @@ -104,6 +108,8 @@ public class SystemUtils { public static final int WINDOWS = 2; public static final int OS_TYPE; + + public static Boolean containerized = null; static { String osName = SystemUtils.osName.toLowerCase(); @@ -273,6 +279,14 @@ public static long osFreePhysicalMemory() { * @return the amount of available physical memory */ public static long osAvailableMemory() { + if(containerized == null){ + containerized = isContainerized(); + } + + if(containerized) { + return osFreePhysicalMemory(); + } + return Pointer.availablePhysicalBytes(); } @@ -715,5 +729,49 @@ private static HotSpotDiagnosticMXBean getHotspotMBean() { } return hotspotMBean; } + + public static boolean isContainerized() { + try { + + // 1. Check for .dockerenv file + if (new File("/.dockerenv").exists()) { + logger.debug("Container detected via .dockerenv file"); + return true; + } + + // 2. Check cgroup info + Path cgroupPath = Paths.get("/proc/self/cgroup"); + if (Files.exists(cgroupPath)) { + List cgroupContent = Files.readAllLines(cgroupPath); + for (String line : cgroupContent) { + if (line.contains("/docker") || + line.contains("/lxc") || + line.contains("/kubepods") || + line.contains("/containerd")) { + logger.debug("Container detected via cgroup: {}", line); + return true; + } + } + } + + // 3. Check mount namespaces + Path mountInfo = Paths.get("/proc/self/mountinfo"); + if (Files.exists(mountInfo)) { + List mountContent = Files.readAllLines(mountInfo); + for (String line : mountContent) { + if (line.contains("/docker") || line.contains("/lxc")) { + logger.debug("Container detected via mountinfo: {}", line); + return true; + } + } + } + + return false; + + } catch (Exception e) { + logger.debug("Error during container detection: {}", e.getMessage()); + return false; + } + } } From bae1bbc7634d0ddcbb0ef9028b0faeb00a50d98c Mon Sep 17 00:00:00 2001 From: lastpeony Date: Fri, 25 Oct 2024 21:27:12 +0300 Subject: [PATCH 2/7] unit test --- src/main/java/io/antmedia/SystemUtils.java | 25 ++-- .../test/statistic/StatsCollectorTest.java | 109 ++++++++++++++++-- 2 files changed, 110 insertions(+), 24 deletions(-) diff --git a/src/main/java/io/antmedia/SystemUtils.java b/src/main/java/io/antmedia/SystemUtils.java index 6ad4e4cec..cd1f0e264 100644 --- a/src/main/java/io/antmedia/SystemUtils.java +++ b/src/main/java/io/antmedia/SystemUtils.java @@ -286,8 +286,8 @@ public static long osAvailableMemory() { if(containerized) { return osFreePhysicalMemory(); } - - return Pointer.availablePhysicalBytes(); + + return availablePhysicalBytes(); } /** @@ -712,7 +712,6 @@ public static void getHeapDump(String filepath) { } } - private static HotSpotDiagnosticMXBean getHotspotMBean() { try { synchronized (SystemUtils.class) { @@ -730,15 +729,21 @@ private static HotSpotDiagnosticMXBean getHotspotMBean() { return hotspotMBean; } + public static long availablePhysicalBytes(){ + return Pointer.availablePhysicalBytes(); + } + public static boolean isContainerized() { try { + Path dockerEnvPath = Paths.get("/.dockerenv"); // 1. Check for .dockerenv file - if (new File("/.dockerenv").exists()) { + if (Files.exists(dockerEnvPath)) { logger.debug("Container detected via .dockerenv file"); return true; } + // 2. Check cgroup info Path cgroupPath = Paths.get("/proc/self/cgroup"); if (Files.exists(cgroupPath)) { @@ -754,18 +759,6 @@ public static boolean isContainerized() { } } - // 3. Check mount namespaces - Path mountInfo = Paths.get("/proc/self/mountinfo"); - if (Files.exists(mountInfo)) { - List mountContent = Files.readAllLines(mountInfo); - for (String line : mountContent) { - if (line.contains("/docker") || line.contains("/lxc")) { - logger.debug("Container detected via mountinfo: {}", line); - return true; - } - } - } - return false; } catch (Exception e) { diff --git a/src/test/java/io/antmedia/test/statistic/StatsCollectorTest.java b/src/test/java/io/antmedia/test/statistic/StatsCollectorTest.java index 4ee9c391a..e8cb929f0 100644 --- a/src/test/java/io/antmedia/test/statistic/StatsCollectorTest.java +++ b/src/test/java/io/antmedia/test/statistic/StatsCollectorTest.java @@ -7,15 +7,16 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.management.ThreadInfo; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.*; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -25,10 +26,12 @@ import org.apache.kafka.clients.producer.RecordMetadata; import org.awaitility.Awaitility; import org.bytedeco.ffmpeg.avutil.AVRational; +import org.bytedeco.javacpp.Pointer; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.red5.server.Launcher; import org.red5.server.api.IContext; @@ -348,7 +351,7 @@ public void testHeartbeat() { return true; }); - Mockito.verify(resMonitor, Mockito.times(1)).startAnalytic(); + Mockito.verify(resMonitor, times(1)).startAnalytic(); resMonitor.cancelHeartBeat(); @@ -357,7 +360,7 @@ public void testHeartbeat() { resMonitor.setHeartBeatEnabled(false); resMonitor.start(); assertFalse(resMonitor.isHeartBeatEnabled()); - Mockito.verify(resMonitor, Mockito.times(1)).startAnalytic(); + Mockito.verify(resMonitor, times(1)).startAnalytic(); resMonitor.cancelHeartBeat(); @@ -481,7 +484,7 @@ public void testCollectAndSendWebRTCStats() { resMonitor.collectAndSendWebRTCClientsStats(); - verify(kafkaProducer, Mockito.times(1)).send(Mockito.any()); + verify(kafkaProducer, times(1)).send(Mockito.any()); } @@ -712,5 +715,95 @@ public void testGetAppAdaptor() } - + + @Test + public void testOsAvailableMemory() { + long availableMemory = 1024; + long containerizedMemory = 2048; + try (MockedStatic mockedSystemUtils = Mockito.mockStatic(SystemUtils.class)) { + // Set up base mocking to allow real method call + mockedSystemUtils.when(SystemUtils::osAvailableMemory) + .thenCallRealMethod(); + + // Test non-containerized scenario + mockedSystemUtils.when(SystemUtils::isContainerized).thenReturn(false); + mockedSystemUtils.when(SystemUtils::availablePhysicalBytes).thenReturn(availableMemory); + + long nonContainerizedResult = SystemUtils.osAvailableMemory(); + assertEquals(availableMemory, nonContainerizedResult); + + // Reset the state for containerized scenario + SystemUtils.containerized = null; + + // Test containerized scenario + mockedSystemUtils.when(SystemUtils::isContainerized).thenReturn(true); + mockedSystemUtils.when(SystemUtils::osFreePhysicalMemory).thenReturn(containerizedMemory); + + long containerizedResult = SystemUtils.osAvailableMemory(); + assertEquals(containerizedMemory, containerizedResult); + + // Verify all calls + mockedSystemUtils.verify(SystemUtils::isContainerized, times(2)); + mockedSystemUtils.verify(SystemUtils::availablePhysicalBytes, times(1)); + mockedSystemUtils.verify(SystemUtils::osFreePhysicalMemory, times(1)); + } + } + + @Test + public void testIsContainerized() { + Path mockDockerEnvPath = Path.of("/tmp/test/.dockerenv"); + Path mockCgroupPath = Path.of("/tmp/test/cgroup"); + + try (MockedStatic mockedFiles = Mockito.mockStatic(Files.class); + MockedStatic mockedPaths = Mockito.mockStatic(Paths.class)) { + + // Setup path mocks + mockedPaths.when(() -> Paths.get("/.dockerenv")).thenReturn(mockDockerEnvPath); + mockedPaths.when(() -> Paths.get("/proc/self/cgroup")).thenReturn(mockCgroupPath); + + // Test 1: Docker environment file exists + mockedFiles.when(() -> Files.exists(mockDockerEnvPath)).thenReturn(true); + assertTrue(SystemUtils.isContainerized()); + + // Test 2: Docker in cgroup + mockedFiles.when(() -> Files.exists(mockDockerEnvPath)).thenReturn(false); + mockedFiles.when(() -> Files.exists(mockCgroupPath)).thenReturn(true); + mockedFiles.when(() -> Files.readAllLines(mockCgroupPath)) + .thenReturn(Collections.singletonList("12:memory:/docker/someId")); + assertTrue(SystemUtils.isContainerized()); + + // Test 3: LXC in cgroup + mockedFiles.when(() -> Files.readAllLines(mockCgroupPath)) + .thenReturn(Collections.singletonList("12:memory:/lxc/someId")); + assertTrue(SystemUtils.isContainerized()); + + // Test 4: Kubernetes in cgroup + mockedFiles.when(() -> Files.readAllLines(mockCgroupPath)) + .thenReturn(Collections.singletonList("12:memory:/kubepods/someId")); + assertTrue(SystemUtils.isContainerized()); + + // Test 5: Containerd in cgroup + mockedFiles.when(() -> Files.readAllLines(mockCgroupPath)) + .thenReturn(Collections.singletonList("12:memory:/containerd/someId")); + assertTrue(SystemUtils.isContainerized()); + + // Test 6: No container + mockedFiles.when(() -> Files.readAllLines(mockCgroupPath)) + .thenReturn(Collections.singletonList("12:memory:/user.slice")); + assertFalse(SystemUtils.isContainerized()); + + // Test 7: File access exception + mockedFiles.when(() -> Files.exists(mockDockerEnvPath)) + .thenThrow(new SecurityException("Access denied")); + assertFalse(SystemUtils.isContainerized()); + + // Test 8: Read exception + mockedFiles.when(() -> Files.exists(mockDockerEnvPath)).thenReturn(false); + mockedFiles.when(() -> Files.exists(mockCgroupPath)).thenReturn(true); + mockedFiles.when(() -> Files.readAllLines(mockCgroupPath)) + .thenThrow(new IOException("Read error")); + assertFalse(SystemUtils.isContainerized()); + } + } + } From 7f28e0703aa52d09d6f328cc7e296bd2d6c98449 Mon Sep 17 00:00:00 2001 From: lastpeony Date: Sat, 26 Oct 2024 01:46:20 +0300 Subject: [PATCH 3/7] use cgroup to calculate mem available in containers --- src/main/java/io/antmedia/SystemUtils.java | 42 ++++++++- .../test/statistic/StatsCollectorTest.java | 86 +++++++++++++++++-- 2 files changed, 119 insertions(+), 9 deletions(-) diff --git a/src/main/java/io/antmedia/SystemUtils.java b/src/main/java/io/antmedia/SystemUtils.java index cd1f0e264..6341c180d 100644 --- a/src/main/java/io/antmedia/SystemUtils.java +++ b/src/main/java/io/antmedia/SystemUtils.java @@ -1,6 +1,8 @@ package io.antmedia; +import java.io.BufferedReader; import java.io.File; +import java.io.FileReader; import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.management.OperatingSystemMXBean; @@ -284,8 +286,14 @@ public static long osAvailableMemory() { } if(containerized) { - return osFreePhysicalMemory(); - } + try{ + return getMemAvailableFromCgroup(); + + }catch (IOException e) { + logger.debug("Could not get mem available from cgroup. Will return os free physical memory instead."); + return osFreePhysicalMemory(); + } + } return availablePhysicalBytes(); } @@ -766,5 +774,35 @@ public static boolean isContainerized() { return false; } } + + public static Long getMemAvailableFromCgroup() throws IOException { + Long memoryUsage; + Long memoryLimit; + + // Try reading memory usage and limit for cgroups v1 + if (Files.exists(Paths.get("/sys/fs/cgroup/memory/memory.usage_in_bytes")) && + Files.exists(Paths.get("/sys/fs/cgroup/memory/memory.limit_in_bytes"))) { + memoryUsage = readLongFromFile("/sys/fs/cgroup/memory/memory.usage_in_bytes"); + memoryLimit = readLongFromFile("/sys/fs/cgroup/memory/memory.limit_in_bytes"); + + // Try reading memory usage and limit for cgroups v2 + } else if (Files.exists(Paths.get("/sys/fs/cgroup/memory.current")) && + Files.exists(Paths.get("/sys/fs/cgroup/memory.max"))) { + memoryUsage = readLongFromFile("/sys/fs/cgroup/memory.current"); + memoryLimit = readLongFromFile("/sys/fs/cgroup/memory.max"); + } else { + logger.debug("Could not find cgroup memory files. Will return os free physical memory instead."); + return osFreePhysicalMemory(); + } + + // Calculate available memory + return memoryLimit - memoryUsage; + } + + public static Long readLongFromFile(String filePath) throws IOException { + try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) { + return Long.parseLong(reader.readLine().trim()); + } + } } diff --git a/src/test/java/io/antmedia/test/statistic/StatsCollectorTest.java b/src/test/java/io/antmedia/test/statistic/StatsCollectorTest.java index e8cb929f0..120a0c9a7 100644 --- a/src/test/java/io/antmedia/test/statistic/StatsCollectorTest.java +++ b/src/test/java/io/antmedia/test/statistic/StatsCollectorTest.java @@ -7,8 +7,7 @@ import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.*; import java.io.IOException; import java.lang.management.ManagementFactory; @@ -720,7 +719,7 @@ public void testGetAppAdaptor() public void testOsAvailableMemory() { long availableMemory = 1024; long containerizedMemory = 2048; - try (MockedStatic mockedSystemUtils = Mockito.mockStatic(SystemUtils.class)) { + try (MockedStatic mockedSystemUtils = mockStatic(SystemUtils.class)) { // Set up base mocking to allow real method call mockedSystemUtils.when(SystemUtils::osAvailableMemory) .thenCallRealMethod(); @@ -737,7 +736,7 @@ public void testOsAvailableMemory() { // Test containerized scenario mockedSystemUtils.when(SystemUtils::isContainerized).thenReturn(true); - mockedSystemUtils.when(SystemUtils::osFreePhysicalMemory).thenReturn(containerizedMemory); + mockedSystemUtils.when(SystemUtils::getMemAvailableFromCgroup).thenReturn(containerizedMemory); long containerizedResult = SystemUtils.osAvailableMemory(); assertEquals(containerizedMemory, containerizedResult); @@ -745,7 +744,7 @@ public void testOsAvailableMemory() { // Verify all calls mockedSystemUtils.verify(SystemUtils::isContainerized, times(2)); mockedSystemUtils.verify(SystemUtils::availablePhysicalBytes, times(1)); - mockedSystemUtils.verify(SystemUtils::osFreePhysicalMemory, times(1)); + mockedSystemUtils.verify(SystemUtils::getMemAvailableFromCgroup, times(1)); } } @@ -754,8 +753,8 @@ public void testIsContainerized() { Path mockDockerEnvPath = Path.of("/tmp/test/.dockerenv"); Path mockCgroupPath = Path.of("/tmp/test/cgroup"); - try (MockedStatic mockedFiles = Mockito.mockStatic(Files.class); - MockedStatic mockedPaths = Mockito.mockStatic(Paths.class)) { + try (MockedStatic mockedFiles = mockStatic(Files.class); + MockedStatic mockedPaths = mockStatic(Paths.class)) { // Setup path mocks mockedPaths.when(() -> Paths.get("/.dockerenv")).thenReturn(mockDockerEnvPath); @@ -806,4 +805,77 @@ public void testIsContainerized() { } } + @Test + public void testGetMemAvailableFromCgroup() throws IOException { + String cgroupV1UsagePath = "/sys/fs/cgroup/memory/memory.usage_in_bytes"; + String cgroupV1LimitPath = "/sys/fs/cgroup/memory/memory.limit_in_bytes"; + String cgroupV2UsagePath = "/sys/fs/cgroup/memory.current"; + String cgroupV2LimitPath = "/sys/fs/cgroup/memory.max"; + long expectedUsage = 5000L; + long expectedLimit = 10000L; + long expectedOsFreeMemory = 8000L; + + // Test all three scenarios using nested try-with-resources + try (MockedStatic mockedFiles = mockStatic(Files.class); + MockedStatic mockedPaths = mockStatic(Paths.class)) { + + // Mock Path objects + Path cgroupV1UsagePathObj = mock(Path.class); + Path cgroupV1LimitPathObj = mock(Path.class); + Path cgroupV2UsagePathObj = mock(Path.class); + Path cgroupV2LimitPathObj = mock(Path.class); + + // Set up Paths.get() mocks + when(Paths.get(cgroupV1UsagePath)).thenReturn(cgroupV1UsagePathObj); + when(Paths.get(cgroupV1LimitPath)).thenReturn(cgroupV1LimitPathObj); + when(Paths.get(cgroupV2UsagePath)).thenReturn(cgroupV2UsagePathObj); + when(Paths.get(cgroupV2LimitPath)).thenReturn(cgroupV2LimitPathObj); + + // Test Scenario 1: cgroups v1 exists + mockedFiles.when(() -> Files.exists(cgroupV1UsagePathObj)).thenReturn(true); + mockedFiles.when(() -> Files.exists(cgroupV1LimitPathObj)).thenReturn(true); + + try (MockedStatic mockedMemoryUtils = mockStatic(SystemUtils.class, + CALLS_REAL_METHODS)) { + mockedMemoryUtils.when(() -> SystemUtils.readLongFromFile(cgroupV1UsagePath)) + .thenReturn(expectedUsage); + mockedMemoryUtils.when(() -> SystemUtils.readLongFromFile(cgroupV1LimitPath)) + .thenReturn(expectedLimit); + + long result = SystemUtils.getMemAvailableFromCgroup(); + assertEquals(expectedLimit - expectedUsage, result); + } + + // Test Scenario 2: cgroups v2 exists (v1 doesn't exist) + mockedFiles.when(() -> Files.exists(cgroupV1UsagePathObj)).thenReturn(false); + mockedFiles.when(() -> Files.exists(cgroupV1LimitPathObj)).thenReturn(false); + mockedFiles.when(() -> Files.exists(cgroupV2UsagePathObj)).thenReturn(true); + mockedFiles.when(() -> Files.exists(cgroupV2LimitPathObj)).thenReturn(true); + + try (MockedStatic mockedMemoryUtils = mockStatic(SystemUtils.class, + CALLS_REAL_METHODS)) { + mockedMemoryUtils.when(() -> SystemUtils.readLongFromFile(cgroupV2UsagePath)) + .thenReturn(expectedUsage); + mockedMemoryUtils.when(() -> SystemUtils.readLongFromFile(cgroupV2LimitPath)) + .thenReturn(expectedLimit); + + long result = SystemUtils.getMemAvailableFromCgroup(); + assertEquals(expectedLimit - expectedUsage, result); + } + + // Test Scenario 3: No cgroups exist, falls back to OS memory + mockedFiles.when(() -> Files.exists(cgroupV2UsagePathObj)).thenReturn(false); + mockedFiles.when(() -> Files.exists(cgroupV2LimitPathObj)).thenReturn(false); + + try (MockedStatic mockedMemoryUtils = mockStatic(SystemUtils.class, + CALLS_REAL_METHODS)) { + mockedMemoryUtils.when(SystemUtils::osFreePhysicalMemory) + .thenReturn(expectedOsFreeMemory); + + long result = SystemUtils.getMemAvailableFromCgroup(); + assertEquals(expectedOsFreeMemory, result); + } + } + } + } From 309e2b5d0caab92dee945aee98ece28c07d34f39 Mon Sep 17 00:00:00 2001 From: lastpeony Date: Tue, 29 Oct 2024 17:04:06 +0300 Subject: [PATCH 4/7] cgroup better handling --- src/main/java/io/antmedia/SystemUtils.java | 73 ++++++++++++-- .../test/statistic/StatsCollectorTest.java | 99 +++++++++++++++++-- 2 files changed, 156 insertions(+), 16 deletions(-) diff --git a/src/main/java/io/antmedia/SystemUtils.java b/src/main/java/io/antmedia/SystemUtils.java index 6341c180d..f4a146f81 100644 --- a/src/main/java/io/antmedia/SystemUtils.java +++ b/src/main/java/io/antmedia/SystemUtils.java @@ -68,6 +68,8 @@ public class SystemUtils { public static final String HEAPDUMP_HPROF = "heapdump.hprof"; + public static final long MAX_CONTAINER_MEMORY_LIMIT_BYTES = 109951162777600L; //100TB + public static final String MAX_MEMORY_CGROUP_V2 = "max"; /** * Obtain Operating System's name. @@ -235,6 +237,25 @@ public static long osCommittedVirtualMemory() { * @return bytes size */ public static long osTotalPhysicalMemory() { + if(containerized == null){ + containerized = isContainerized(); + } + + if(containerized) { + try{ + return getMemoryLimitFromCgroup(); + + }catch (IOException e) { + logger.debug("Could not get memory limit from c group. {}", e.getMessage()); + } + } + + return getTotalPhysicalMemorySize(); + + } + + public static long getTotalPhysicalMemorySize(){ + final OperatingSystemMXBean osBean = ManagementFactory.getOperatingSystemMXBean(); if (osBean instanceof UnixOperatingSystemMXBean) { return (((UnixOperatingSystemMXBean) osBean).getTotalPhysicalMemorySize()); @@ -248,8 +269,10 @@ public static long osTotalPhysicalMemory() { return -1L; } } + } + /** * Obtain Free Physical Memory from Operating System's RAM. * @@ -775,6 +798,40 @@ public static boolean isContainerized() { } } + public static Long getMemoryLimitFromCgroup() throws IOException { + long memoryLimit; + + // Try reading memory limit for cgroups v1 + if (Files.exists(Paths.get("/sys/fs/cgroup/memory/memory.limit_in_bytes"))) { + String memoryLimitString = readCgroupFile("/sys/fs/cgroup/memory/memory.limit_in_bytes"); + + memoryLimit = Long.parseLong(memoryLimitString); + + //in cgroups v1 if memory limit is not set on container this value returns 9223372036854771712 2^63-1, but number is not 100%, changes based on architecture + //if memory limit returned is higher than 100TB its not set using cgroups. + if(memoryLimit > MAX_CONTAINER_MEMORY_LIMIT_BYTES) { + memoryLimit = getTotalPhysicalMemorySize(); + } + + // Try reading memory limit for cgroups v2 + } else if (Files.exists(Paths.get("/sys/fs/cgroup/memory.max"))) { + String memoryLimitString = readCgroupFile("/sys/fs/cgroup/memory.max"); + + if(MAX_MEMORY_CGROUP_V2.equals(memoryLimitString)){ // memory limit is not set using cgroups v2 + memoryLimit = getTotalPhysicalMemorySize(); + }else{ + memoryLimit = Long.parseLong(memoryLimitString); + } + + + } else { + logger.debug("Could not find cgroup max memory file. Will return os physical memory instead."); + return getTotalPhysicalMemorySize(); + } + + return memoryLimit; + } + public static Long getMemAvailableFromCgroup() throws IOException { Long memoryUsage; Long memoryLimit; @@ -782,14 +839,18 @@ public static Long getMemAvailableFromCgroup() throws IOException { // Try reading memory usage and limit for cgroups v1 if (Files.exists(Paths.get("/sys/fs/cgroup/memory/memory.usage_in_bytes")) && Files.exists(Paths.get("/sys/fs/cgroup/memory/memory.limit_in_bytes"))) { - memoryUsage = readLongFromFile("/sys/fs/cgroup/memory/memory.usage_in_bytes"); - memoryLimit = readLongFromFile("/sys/fs/cgroup/memory/memory.limit_in_bytes"); + String memoryUsageString = readCgroupFile("/sys/fs/cgroup/memory/memory.usage_in_bytes"); + memoryUsage = Long.parseLong(memoryUsageString); + memoryLimit = getMemoryLimitFromCgroup(); // Try reading memory usage and limit for cgroups v2 } else if (Files.exists(Paths.get("/sys/fs/cgroup/memory.current")) && Files.exists(Paths.get("/sys/fs/cgroup/memory.max"))) { - memoryUsage = readLongFromFile("/sys/fs/cgroup/memory.current"); - memoryLimit = readLongFromFile("/sys/fs/cgroup/memory.max"); + + String memoryUsageString = readCgroupFile("/sys/fs/cgroup/memory.current"); + memoryUsage = Long.parseLong(memoryUsageString); + memoryLimit = getMemoryLimitFromCgroup(); + } else { logger.debug("Could not find cgroup memory files. Will return os free physical memory instead."); return osFreePhysicalMemory(); @@ -799,9 +860,9 @@ public static Long getMemAvailableFromCgroup() throws IOException { return memoryLimit - memoryUsage; } - public static Long readLongFromFile(String filePath) throws IOException { + public static String readCgroupFile(String filePath) throws IOException { try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) { - return Long.parseLong(reader.readLine().trim()); + return reader.readLine().trim(); } } diff --git a/src/test/java/io/antmedia/test/statistic/StatsCollectorTest.java b/src/test/java/io/antmedia/test/statistic/StatsCollectorTest.java index 120a0c9a7..98d1d1e66 100644 --- a/src/test/java/io/antmedia/test/statistic/StatsCollectorTest.java +++ b/src/test/java/io/antmedia/test/statistic/StatsCollectorTest.java @@ -811,6 +811,8 @@ public void testGetMemAvailableFromCgroup() throws IOException { String cgroupV1LimitPath = "/sys/fs/cgroup/memory/memory.limit_in_bytes"; String cgroupV2UsagePath = "/sys/fs/cgroup/memory.current"; String cgroupV2LimitPath = "/sys/fs/cgroup/memory.max"; + String expectedUsageStr = "5000"; + String expectedLimitStr = "10000"; long expectedUsage = 5000L; long expectedLimit = 10000L; long expectedOsFreeMemory = 8000L; @@ -837,13 +839,13 @@ public void testGetMemAvailableFromCgroup() throws IOException { try (MockedStatic mockedMemoryUtils = mockStatic(SystemUtils.class, CALLS_REAL_METHODS)) { - mockedMemoryUtils.when(() -> SystemUtils.readLongFromFile(cgroupV1UsagePath)) - .thenReturn(expectedUsage); - mockedMemoryUtils.when(() -> SystemUtils.readLongFromFile(cgroupV1LimitPath)) - .thenReturn(expectedLimit); + mockedMemoryUtils.when(() -> SystemUtils.readCgroupFile(cgroupV1UsagePath)) + .thenReturn(expectedUsageStr); + mockedMemoryUtils.when(() -> SystemUtils.readCgroupFile(cgroupV1LimitPath)) + .thenReturn(expectedLimitStr); long result = SystemUtils.getMemAvailableFromCgroup(); - assertEquals(expectedLimit - expectedUsage, result); + assertEquals(Long.parseLong(expectedLimitStr) - Long.parseLong(expectedUsageStr), result); } // Test Scenario 2: cgroups v2 exists (v1 doesn't exist) @@ -854,13 +856,13 @@ public void testGetMemAvailableFromCgroup() throws IOException { try (MockedStatic mockedMemoryUtils = mockStatic(SystemUtils.class, CALLS_REAL_METHODS)) { - mockedMemoryUtils.when(() -> SystemUtils.readLongFromFile(cgroupV2UsagePath)) - .thenReturn(expectedUsage); - mockedMemoryUtils.when(() -> SystemUtils.readLongFromFile(cgroupV2LimitPath)) - .thenReturn(expectedLimit); + mockedMemoryUtils.when(() -> SystemUtils.readCgroupFile(cgroupV2UsagePath)) + .thenReturn(expectedUsageStr); + mockedMemoryUtils.when(() -> SystemUtils.readCgroupFile(cgroupV2LimitPath)) + .thenReturn(expectedLimitStr); long result = SystemUtils.getMemAvailableFromCgroup(); - assertEquals(expectedLimit - expectedUsage, result); + assertEquals(Long.parseLong(expectedLimitStr) - Long.parseLong(expectedUsageStr), result); } // Test Scenario 3: No cgroups exist, falls back to OS memory @@ -878,4 +880,81 @@ public void testGetMemAvailableFromCgroup() throws IOException { } } + @Test + public void testOsTotalPhysicalMemory() throws IOException { + final long PHYSICAL_MEMORY_SIZE = 16_000_000_000L; // 16GB + + try (MockedStatic mockedFiles = mockStatic(Files.class); + MockedStatic mockedSystemUtils = mockStatic(SystemUtils.class, + CALLS_REAL_METHODS)) { + + // Test Case 1: Non-containerized environment + mockedSystemUtils.when(SystemUtils::isContainerized) + .thenReturn(false); + mockedSystemUtils.when(SystemUtils::getTotalPhysicalMemorySize) + .thenReturn(PHYSICAL_MEMORY_SIZE); + + // Call the real method for osTotalPhysicalMemory + mockedSystemUtils.when(SystemUtils::osTotalPhysicalMemory) + .thenCallRealMethod(); + + assertEquals(PHYSICAL_MEMORY_SIZE, SystemUtils.osTotalPhysicalMemory()); + + // Test Case 2: Containerized environment with cgroups v1 + mockedSystemUtils.when(SystemUtils::isContainerized) + .thenReturn(true); + + // Mock cgroups v1 path exists + mockedFiles.when(() -> Files.exists(Paths.get("/sys/fs/cgroup/memory/memory.limit_in_bytes"))) + .thenReturn(true); + mockedFiles.when(() -> Files.exists(Paths.get("/sys/fs/cgroup/memory.max"))) + .thenReturn(false); + + // Test 2a: Normal memory limit + long containerLimit = 8_000_000_000L; // 8GB + mockedSystemUtils.when(() -> SystemUtils.readCgroupFile(anyString())) + .thenReturn(String.valueOf(containerLimit)); + + assertEquals(containerLimit, SystemUtils.getMemoryLimitFromCgroup().longValue()); + + // Test 2b: Memory limit above MAX_CONTAINER_MEMORY_LIMIT_BYTES + mockedSystemUtils.when(() -> SystemUtils.readCgroupFile(anyString())) + .thenReturn(String.valueOf(SystemUtils.MAX_CONTAINER_MEMORY_LIMIT_BYTES + 1)); + + assertEquals(PHYSICAL_MEMORY_SIZE, SystemUtils.getMemoryLimitFromCgroup().longValue()); + + // Test Case 3: Containerized environment with cgroups v2 + mockedFiles.when(() -> Files.exists(Paths.get("/sys/fs/cgroup/memory/memory.limit_in_bytes"))) + .thenReturn(false); + mockedFiles.when(() -> Files.exists(Paths.get("/sys/fs/cgroup/memory.max"))) + .thenReturn(true); + + // Test 3a: Normal memory limit + mockedSystemUtils.when(() -> SystemUtils.readCgroupFile(anyString())) + .thenReturn("4000000000"); // 4GB + + assertEquals(4_000_000_000L, SystemUtils.getMemoryLimitFromCgroup().longValue()); + + // Test 3b: No limit set (max) + mockedSystemUtils.when(() -> SystemUtils.readCgroupFile(anyString())) + .thenReturn(SystemUtils.MAX_MEMORY_CGROUP_V2); + + assertEquals(PHYSICAL_MEMORY_SIZE, SystemUtils.getMemoryLimitFromCgroup().longValue()); + + // Test Case 4: No cgroup files exist + mockedFiles.when(() -> Files.exists(any(Path.class))) + .thenReturn(false); + + assertEquals(PHYSICAL_MEMORY_SIZE, SystemUtils.getMemoryLimitFromCgroup().longValue()); + + // Test Case 5: IOException handling + mockedSystemUtils.when(SystemUtils::isContainerized) + .thenReturn(true); + mockedSystemUtils.when(SystemUtils::getMemoryLimitFromCgroup) + .thenThrow(new IOException("Test exception")); + + assertEquals(PHYSICAL_MEMORY_SIZE, SystemUtils.osTotalPhysicalMemory()); + } + } + } From 60d9681cc546d0f9185edcbcc721205c341b08f3 Mon Sep 17 00:00:00 2001 From: lastpeony Date: Tue, 29 Oct 2024 18:11:04 +0300 Subject: [PATCH 5/7] lxc is container bug fix --- src/main/java/io/antmedia/SystemUtils.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/io/antmedia/SystemUtils.java b/src/main/java/io/antmedia/SystemUtils.java index f4a146f81..a9a1bfecd 100644 --- a/src/main/java/io/antmedia/SystemUtils.java +++ b/src/main/java/io/antmedia/SystemUtils.java @@ -780,10 +780,10 @@ public static boolean isContainerized() { if (Files.exists(cgroupPath)) { List cgroupContent = Files.readAllLines(cgroupPath); for (String line : cgroupContent) { - if (line.contains("/docker") || - line.contains("/lxc") || - line.contains("/kubepods") || - line.contains("/containerd")) { + if (line.contains("docker") || + line.contains("lxc") || + line.contains("kubepods") || + line.contains("containerd")) { logger.debug("Container detected via cgroup: {}", line); return true; } From 8aa05289cca9cefca4f66048354f127aaeb95ea6 Mon Sep 17 00:00:00 2001 From: lastpeony Date: Wed, 30 Oct 2024 13:54:24 +0300 Subject: [PATCH 6/7] env variable check --- src/main/java/io/antmedia/SystemUtils.java | 24 ++++++++++++------- .../test/statistic/StatsCollectorTest.java | 8 ++++--- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/main/java/io/antmedia/SystemUtils.java b/src/main/java/io/antmedia/SystemUtils.java index a9a1bfecd..57087046c 100644 --- a/src/main/java/io/antmedia/SystemUtils.java +++ b/src/main/java/io/antmedia/SystemUtils.java @@ -14,6 +14,7 @@ import javax.management.MBeanServer; +import org.apache.commons.lang3.StringUtils; import org.bytedeco.javacpp.Pointer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -774,8 +775,14 @@ public static boolean isContainerized() { return true; } + // 2. Check env variable + String container = System.getenv("container"); + if(StringUtils.isNotBlank(container)){ + logger.debug("Container detected via env variable."); + return true; + } - // 2. Check cgroup info + // 3. Check cgroup info Path cgroupPath = Paths.get("/proc/self/cgroup"); if (Files.exists(cgroupPath)) { List cgroupContent = Files.readAllLines(cgroupPath); @@ -790,12 +797,12 @@ public static boolean isContainerized() { } } - return false; - } catch (Exception e) { - logger.debug("Error during container detection: {}", e.getMessage()); - return false; + logger.error("Error during container detection: {}", e.getMessage()); + } + + return false; } public static Long getMemoryLimitFromCgroup() throws IOException { @@ -807,9 +814,10 @@ public static Long getMemoryLimitFromCgroup() throws IOException { memoryLimit = Long.parseLong(memoryLimitString); - //in cgroups v1 if memory limit is not set on container this value returns 9223372036854771712 2^63-1, but number is not 100%, changes based on architecture - //if memory limit returned is higher than 100TB its not set using cgroups. - if(memoryLimit > MAX_CONTAINER_MEMORY_LIMIT_BYTES) { + // In cgroups v1, if the memory limit for a container isn't set, it typically returns 9223372036854771712 (2^63-1). + // However, this value may vary based on the architecture. + // Therefore, we consider it acceptable if the returned memory limit exceeds 100TB, indicating that the limit is not configured using cgroups. + if(memoryLimit > MAX_CONTAINER_MEMORY_LIMIT_BYTES || memoryLimit == 0 || memoryLimit == -1) { memoryLimit = getTotalPhysicalMemorySize(); } diff --git a/src/test/java/io/antmedia/test/statistic/StatsCollectorTest.java b/src/test/java/io/antmedia/test/statistic/StatsCollectorTest.java index 98d1d1e66..72c891782 100644 --- a/src/test/java/io/antmedia/test/statistic/StatsCollectorTest.java +++ b/src/test/java/io/antmedia/test/statistic/StatsCollectorTest.java @@ -754,7 +754,8 @@ public void testIsContainerized() { Path mockCgroupPath = Path.of("/tmp/test/cgroup"); try (MockedStatic mockedFiles = mockStatic(Files.class); - MockedStatic mockedPaths = mockStatic(Paths.class)) { + MockedStatic mockedPaths = mockStatic(Paths.class); + ) { // Setup path mocks mockedPaths.when(() -> Paths.get("/.dockerenv")).thenReturn(mockDockerEnvPath); @@ -764,6 +765,7 @@ public void testIsContainerized() { mockedFiles.when(() -> Files.exists(mockDockerEnvPath)).thenReturn(true); assertTrue(SystemUtils.isContainerized()); + // Test 2: Docker in cgroup mockedFiles.when(() -> Files.exists(mockDockerEnvPath)).thenReturn(false); mockedFiles.when(() -> Files.exists(mockCgroupPath)).thenReturn(true); @@ -803,6 +805,7 @@ public void testIsContainerized() { .thenThrow(new IOException("Read error")); assertFalse(SystemUtils.isContainerized()); } + } @Test @@ -813,8 +816,7 @@ public void testGetMemAvailableFromCgroup() throws IOException { String cgroupV2LimitPath = "/sys/fs/cgroup/memory.max"; String expectedUsageStr = "5000"; String expectedLimitStr = "10000"; - long expectedUsage = 5000L; - long expectedLimit = 10000L; + long expectedOsFreeMemory = 8000L; // Test all three scenarios using nested try-with-resources From feb0b63516b752a77f89d12468836aef609aaf59 Mon Sep 17 00:00:00 2001 From: lastpeony Date: Wed, 30 Oct 2024 17:21:17 +0300 Subject: [PATCH 7/7] fix unit test --- .../io/antmedia/test/rest/BroadcastRestServiceV2UnitTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/test/java/io/antmedia/test/rest/BroadcastRestServiceV2UnitTest.java b/src/test/java/io/antmedia/test/rest/BroadcastRestServiceV2UnitTest.java index 4e2abbe50..34d437be8 100644 --- a/src/test/java/io/antmedia/test/rest/BroadcastRestServiceV2UnitTest.java +++ b/src/test/java/io/antmedia/test/rest/BroadcastRestServiceV2UnitTest.java @@ -2150,7 +2150,7 @@ public void testAddIPCamera() { streamSourceRest.setAppCtx(appContext); - StatsCollector monitorService = new StatsCollector(); + StatsCollector monitorService = spy(StatsCollector.class); when(appContext.getBean(IStatsCollector.BEAN_NAME)).thenReturn(monitorService); @@ -2181,6 +2181,8 @@ public void testAddIPCamera() { int cpuLoad2 = 70; int cpuLimit2 = 80; + when(monitorService.getMemoryLoad()).thenReturn(20); + monitorService.setCpuLimit(cpuLimit2); monitorService.setCpuLoad(cpuLoad2);