Skip to content

Add container support for server metrics recording. #463

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 136 additions & 0 deletions src/Recorders/Servers.php
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,13 @@ class Servers
*/
protected static $detectMemoryUsing;

/**
* Callback to detect if running in a container.
*
* @var null|(callable(): bool)
*/
protected static $detectContainerUsing;

/**
* The events to listen for.
*
Expand Down Expand Up @@ -66,13 +73,73 @@ public static function detectMemoryUsing(?callable $callback): void
self::$detectMemoryUsing = $callback;
}

/**
* Detect if running in a container via the given callback.
*
* @param null|(callable(): bool) $callback
*/
public static function detectContainerUsing(?callable $callback): void
{
self::$detectContainerUsing = $callback;
}

/**
* Detect if running in a container.
*/
protected function isContainer(): bool
{
if (self::$detectContainerUsing) {
return (self::$detectContainerUsing)();
}

if (file_exists('/.dockerenv') || file_exists('/.dockerinit')) {
return true;
}

if (getenv('KUBERNETES_SERVICE_HOST') !== false) {
return true;
}

if (PHP_OS_FAMILY === 'Linux' && file_exists('/proc/1/cgroup')) {
$content = file_get_contents('/proc/1/cgroup');
if ($content && (str_contains($content, 'docker') ||
strpos($content, 'kubepods') !== false ||
str_contains($content, 'containerd'))) {
return true;
}
}

return false;
}

/**
* Record the system stats.
*/
public function record(SharedBeat $event): void
{
$this->throttle(15, $event, function ($event) {
$server = $this->config->get('pulse.recorders.'.self::class.'.server_name');

if ($this->isContainer()) {
$containerInfo = '';

if (PHP_OS_FAMILY === 'Linux' && file_exists('/proc/1/cgroup')) {
$content = file_get_contents('/proc/1/cgroup');
if ($content) {
preg_match('/[0-9a-f]{12}/', $content, $matches);
if (!empty($matches)) {
$containerInfo = ' [container: ' . substr($matches[0], 0, 12) . ']';
} else {
$containerInfo = ' [containerized]';
}
}
} else {
$containerInfo = ' [containerized]';
}

$server .= $containerInfo;
}

$slug = Str::slug($server);

['total' => $memoryTotal, 'used' => $memoryUsed] = $this->memory();
Expand Down Expand Up @@ -106,6 +173,41 @@ protected function cpu(): int
return (self::$detectCpuUsing)();
}

if ($this->isContainer() && PHP_OS_FAMILY === 'Linux') {
try {
if (file_exists('/sys/fs/cgroup/cpu.stat')) {
$stat = file_get_contents('/sys/fs/cgroup/cpu.stat');
if ($stat !== false) {
preg_match('/usage_usec\s+(\d+)/', $stat, $matches);
if (!empty($matches)) {
$usage1 = (int) $matches[1];
usleep(100000); // 100ms
preg_match('/usage_usec\s+(\d+)/', $stat, $matches);
if (!empty($matches)) {
$usage2 = (int) $matches[1];
$diff = $usage2 - $usage1;
return min(100, (int) ($diff / 1000 / 100));
}
}
}
}

if (file_exists('/sys/fs/cgroup/cpu/cpuacct.usage') || file_exists('/sys/fs/cgroup/cpuacct/cpuacct.usage')) {
$file = file_exists('/sys/fs/cgroup/cpu/cpuacct.usage')
? '/sys/fs/cgroup/cpu/cpuacct.usage'
: '/sys/fs/cgroup/cpuacct/cpuacct.usage';

$usage1 = (int) file_get_contents($file);
usleep(100000); // 100ms
$usage2 = (int) file_get_contents($file);
$diff = $usage2 - $usage1;
return min(100, (int) ($diff / 1000000 / 100));
}
} catch (\Exception) {
throw new RuntimeException('The pulse:check was unable to determine CPU usage in a containerized environment. Please ensure the cgroup files are accessible and valid.');
}
}

return match (PHP_OS_FAMILY) {
'Darwin' => (int) `top -l 1 | grep -E "^CPU" | tail -1 | awk '{ print $3 + $5 }'`,
'Linux' => (int) `top -bn1 | grep -E '^(%Cpu|CPU)' | awk '{ print $2 + $4 }'`,
Expand All @@ -126,6 +228,40 @@ protected function memory(): array
return (self::$detectMemoryUsing)();
}

if ($this->isContainer() && PHP_OS_FAMILY === 'Linux') {
try {
if (file_exists('/sys/fs/cgroup/memory.max') && file_exists('/sys/fs/cgroup/memory.current')) {
$memoryLimit = (int) file_get_contents('/sys/fs/cgroup/memory.max');
$memoryCurrent = (int) file_get_contents('/sys/fs/cgroup/memory.current');

if ($memoryLimit === 0 || $memoryLimit === PHP_INT_MAX || $memoryLimit === -1) {
$memoryLimit = intval(`cat /proc/meminfo | grep MemTotal | grep -E -o '[0-9]+'` * 1024);
}

return [
'total' => intval($memoryLimit / 1024 / 1024),
'used' => intval($memoryCurrent / 1024 / 1024),
];
}

if (file_exists('/sys/fs/cgroup/memory/memory.limit_in_bytes') && file_exists('/sys/fs/cgroup/memory/memory.usage_in_bytes')) {
$memoryLimit = (int) file_get_contents('/sys/fs/cgroup/memory/memory.limit_in_bytes');
$memoryUsage = (int) file_get_contents('/sys/fs/cgroup/memory/memory.usage_in_bytes');

if ($memoryLimit === 0 || $memoryLimit === PHP_INT_MAX || $memoryLimit === -1 || $memoryLimit > 10 * 1024 * 1024 * 1024 * 1024) { // > 10TB
$memoryLimit = intval(`cat /proc/meminfo | grep MemTotal | grep -E -o '[0-9]+'` * 1024);
}

return [
'total' => intval($memoryLimit / 1024 / 1024),
'used' => intval($memoryUsage / 1024 / 1024),
];
}
} catch (\Exception) {
throw new RuntimeException('The pulse:check was unable to determine memory usage in a containerized environment. Please ensure the cgroup files are accessible and valid.');
}
}

$memoryTotal = match (PHP_OS_FAMILY) {
'Darwin' => intval(`sysctl hw.memsize | grep -Eo '[0-9]+'` / 1024 / 1024),
'Linux' => intval(`cat /proc/meminfo | grep MemTotal | grep -E -o '[0-9]+'` / 1024),
Expand Down
52 changes: 52 additions & 0 deletions tests/Feature/Recorders/ServersTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -78,3 +78,55 @@
$payload = json_decode($value->value);
expect($payload->storage)->toHaveCount(1);
});

it('includes container information in server name when in a container', function () {
Config::set('pulse.recorders.'.Servers::class.'.server_name', 'Foo');
Date::setTestNow(Date::now()->startOfMinute());

// Set container detection to return true
Servers::detectContainerUsing(fn () => true);

event(new SharedBeat(CarbonImmutable::now(), 'instance-id'));
Pulse::ingest();

$value = Pulse::ignore(fn () => DB::table('pulse_values')->sole());
$payload = json_decode($value->value);

// Verify server name includes container information
expect($payload->name)->toContain('Foo');
expect($payload->name)->toContain('[containerized]');

// Reset container detection
Servers::detectContainerUsing(null);
});

it('uses container-specific methods for CPU and memory when in a container', function () {
Config::set('pulse.recorders.'.Servers::class.'.server_name', 'Foo');
Date::setTestNow(Date::now()->startOfMinute());

// Set container detection to return true
Servers::detectContainerUsing(fn () => true);

// Set custom CPU and memory detection to simulate container-specific methods
Servers::detectCpuUsing(fn () => 42);
Servers::detectMemoryUsing(fn () => [
'total' => 8192, // 8GB
'used' => 4096, // 4GB
]);

event(new SharedBeat(CarbonImmutable::now(), 'instance-id'));
Pulse::ingest();

$value = Pulse::ignore(fn () => DB::table('pulse_values')->sole());
$payload = json_decode($value->value);

// Verify CPU and memory values
expect($payload->cpu)->toBe(42);
expect($payload->memory_total)->toBe(8192);
expect($payload->memory_used)->toBe(4096);

// Reset container, CPU, and memory detection
Servers::detectContainerUsing(null);
Servers::detectCpuUsing(null);
Servers::detectMemoryUsing(null);
});