alexpires.me/a13labs.infra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

auto_suspend: busctl->shutdown now, 1min timer config, AMD JSON parsing, locale fix, unit tests

Browse Source
This commit is contained in:
Alexandre Pires
2026-06-09 20:45:08 +02:00
parent f8a0a8ab1f
commit 80c154df03
9 changed files with 701 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ansible/roles/auto_suspend/files/auto_suspend.py
+44 -43
View File
@@ -11,7 +11,6 @@ across timer invocations. State is fully reset on suspend resume.
"""
import argparse
import csv
import json
import logging
import logging.handlers
@@ -142,11 +141,17 @@ def measure_cpu(logger):
Runs ``mpstat 1 1`` and extracts the last field of the "Average:" line,
which is ``%idle``. Returns a float, or None on failure.
Uses ``LC_ALL=C`` to force English locale output regardless of system
locale (avoids issues like Portuguese ``Média:`` with comma decimals).
"""
env = dict(os.environ)
env["LC_ALL"] = "C"
try:
result = subprocess.run(
["mpstat", "1", "1"],
capture_output=True, text=True, timeout=15,
env=env,
)
if result.returncode != 0:
logger.warning("mpstat returned non-zero: %s", result.stderr.strip())
@@ -161,6 +166,7 @@ def measure_cpu(logger):
logger.debug("CPU idle: %.1f%%", idle_pct)
return idle_pct
logger.warning("mpstat output did not contain expected data line")
return None
except subprocess.TimeoutExpired:
logger.warning("mpstat timed out")
@@ -198,7 +204,7 @@ def measure_gpu_nvidia(logger):
return None
max_util = max(utils)
logger.debug("NVIDIA GPU utilization: max=%.1f%%", max_util)
logger.info("NVIDIA GPU utilization: max=%.1f%%", max_util)
return max_util
except FileNotFoundError:
logger.debug("nvidia-smi not found")
@@ -215,51 +221,45 @@ def measure_gpu_amd(logger):
"""
Measure GPU utilization via amd-smi.
Parses the ``gfx`` column from ``amd-smi monitor --csv`` by header name
(not by fixed column index). Returns None if amd-smi is unavailable.
Parses the ``gfx.value`` field from ``amd-smi monitor --json``.
Returns the utilization of the first GPU, or None if amd-smi is unavailable.
"""
try:
result = subprocess.run(
["amd-smi", "monitor", "--csv"],
["amd-smi", "monitor", "--json"],
capture_output=True, text=True, timeout=15,
)
if result.returncode != 0:
logger.debug("amd-smi returned %d — no AMD GPU", result.returncode)
return None
lines = result.stdout.strip().splitlines()
if len(lines) < 2:
logger.debug("amd-smi output too short")
raw = result.stdout.strip()
data = json.loads(result.stdout)
if not data or not isinstance(data, list):
logger.debug("amd-smi JSON output empty or invalid")
return None
# Filter out non-data lines (comments, annotations with wrong field count)
csv_reader = csv.reader(lines)
header = next(csv_reader)
for fields in csv_reader:
if len(fields) != len(header):
continue
row = dict(zip(header, fields))
try:
# Try both "gfx" and "gfx_util" column names
gfx_val = row.get("gfx") or row.get("gfx_util")
if gfx_val is None:
logger.debug("amd-smi CSV has no 'gfx' column; columns: %s",
list(row.keys()))
return None
util = float(gfx_val)
logger.debug("AMD GPU utilization: %.1f%%", util)
return util
except (ValueError, TypeError):
continue
return None
gpu = data[0]
gfx = gpu.get("gfx")
if isinstance(gfx, dict):
gfx_val = gfx.get("value")
else:
gfx_val = gfx
if gfx_val is None:
logger.debug("amd-smi JSON has no 'gfx.value' field; fields: %s",
list(gpu.keys()))
return None
util = float(gfx_val)
logger.info("AMD GPU utilization: %.1f%% (gfx=%s)", util, gfx_val)
return util
except FileNotFoundError:
logger.debug("amd-smi not found")
return None
except subprocess.TimeoutExpired:
logger.warning("amd-smi timed out")
return None
except BaseException as exc:
except (json.JSONDecodeError, ValueError, TypeError) as exc:
logger.warning("Failed to parse amd-smi output: %s", exc)
return None
@@ -339,24 +339,25 @@ def update_averages(state, cpu_n, gpu_n, alpha, logger):
# Evaluation
# ---------------------------------------------------------------------------
def get_why_not_idle(state, has_users, logger):
def get_why_not_idle(state, has_users, cpu_idle_threshold=DEFAULT_CPU_IDLE_THRESHOLD,
gpu_util_threshold=DEFAULT_GPU_UTIL_THRESHOLD, min_samples=DEFAULT_MIN_SAMPLES, logger=None):
"""Return a human-readable reason why the system is not idle."""
reasons = []
if has_users:
reasons.append("users logged in")
if state["cpu_avg"] is not None:
if state["cpu_avg"] <= DEFAULT_CPU_IDLE_THRESHOLD:
if state["cpu_avg"] <= cpu_idle_threshold:
reasons.append(
f"CPU idle avg {state['cpu_avg']:.1f}% <= threshold {DEFAULT_CPU_IDLE_THRESHOLD:.0f}%"
f"CPU idle avg {state['cpu_avg']:.1f}% <= threshold {cpu_idle_threshold:.0f}%"
)
if state["gpu_avg"] is not None:
if state["gpu_avg"] >= DEFAULT_GPU_UTIL_THRESHOLD:
if state["gpu_avg"] >= gpu_util_threshold:
reasons.append(
f"GPU util avg {state['gpu_avg']:.1f}% >= threshold {DEFAULT_GPU_UTIL_THRESHOLD:.0f}%"
f"GPU util avg {state['gpu_avg']:.1f}% >= threshold {gpu_util_threshold:.0f}%"
)
if not reasons:
if state["sample_count"] < DEFAULT_MIN_SAMPLES:
reasons.append(f"warming up ({state['sample_count']}/{DEFAULT_MIN_SAMPLES} samples)")
if state["sample_count"] < min_samples:
reasons.append(f"warming up ({state['sample_count']}/{min_samples} samples)")
else:
reasons.append("unknown")
return "; ".join(reasons)
@@ -491,17 +492,17 @@ def main():
try:
result = subprocess.run(
["systemctl", "suspend"],
["shutdown", "now"],
capture_output=True, text=True, timeout=10,
)
if result.returncode != 0:
logger.error("systemctl suspend failed: %s", result.stderr.strip())
logger.error("shutdown failed: %s", result.stderr.strip())
else:
logger.info("systemctl suspend succeeded")
logger.info("shutdown succeeded")
except subprocess.TimeoutExpired:
logger.error("systemctl suspend timed out")
logger.error("shutdown timed out")
except FileNotFoundError:
logger.error("systemctl not found — cannot suspend")
logger.error("shutdown not found — cannot suspend")
except BaseException as exc:
logger.error("Failed to suspend: %s", exc)
@@ -511,7 +512,7 @@ def main():
else:
if not args.dry_run:
state["idle_counter"] = 0
reason = get_why_not_idle(state, has_users, logger)
reason = get_why_not_idle(state, has_users, args.cpu_idle_threshold, args.gpu_util_threshold, args.min_samples, logger)
logger.info("Not idle (%s) — reset counter", reason)
# --- Save and exit ---
ansible/roles/auto_suspend/files/fixtures/amd-smi-monitor--csv.txt
-3
View File
@@ -1,3 +0,0 @@
gpu,power_usage,hotspot_temperature,memory_temperature,gfx,gfx_clock,mem,mem_clock,encoder,decoder,vclock,dclock,single_bit_ecc,double_bit_ecc,pcie_replay,vram_used,vram_total,pcie_bw,pviol,tviol,phot_tviol,vr_tviol,hbm_tviol
0,20,52,60,100,1200,0,96,N/A,0.0,25,25,0,0,N/A,11913,16334,N/A,N/A,N/A,N/A,N/A,N/A
ansible/roles/auto_suspend/files/fixtures/amd-smi-monitor--json.txt
+27
View File
@@ -0,0 +1,27 @@
[
{
"gpu": 0,
"power_usage": {"value": 20, "unit": "W"},
"hotspot_temperature": {"value": 52, "unit": "C"},
"memory_temperature": {"value": 60, "unit": "C"},
"gfx": {"value": 100, "unit": "%"},
"gfx_clock": {"value": 1200, "unit": "MHz"},
"mem": {"value": 0, "unit": "%"},
"mem_clock": {"value": 96, "unit": "MHz"},
"encoder": "N/A",
"decoder": {"value": 0.0, "unit": "%"},
"vclock": {"value": 25, "unit": "MHz"},
"dclock": {"value": 25, "unit": "MHz"},
"single_bit_ecc": 0,
"double_bit_ecc": 0,
"pcie_replay": "N/A",
"vram_used": {"value": 11913, "unit": "MB"},
"vram_total": {"value": 16334, "unit": "MB"},
"pcie_bw": {"value": "N/A", "unit": "Mb/s"},
"pviol": {"value": "N/A", "unit": "%"},
"tviol": {"value": "N/A", "unit": "%"},
"phot_tviol": {"value": "N/A", "unit": "%"},
"vr_tviol": {"value": "N/A", "unit": "%"},
"hbm_tviol": {"value": "N/A", "unit": "%"}
}
]
ansible/roles/auto_suspend/files/test_auto_suspend.py
+99 -12
View File
@@ -48,13 +48,9 @@ NVIDIA_BUSY_4GAPS = """ 0
NVIDIA_BUSY_SINGLE = """ 95
"""
NVIDIA_NONE = ""
AMD_SMII_GFX = _load_fixture("amd-smi-monitor--csv.txt")
AMD_SMII_GFX_0 = """gpu,power_usage,hotspot_temperature,memory_temperature,gfx,gfx_clock,mem,mem_clock,encoder,decoder,vclock,dclock,single_bit_ecc,double_bit_ecc,pcie_replay,vram_used,vram_total,pcie_bw,pviol,tviol,phot_tviol,vr_tviol,hbm_tviol
0,20,52,62,0,1200,0,96,N/A,0.0,25,25,0,0,N/A,11928,16334,N/A,N/A,N/A,N/A,N/A,N/A
"""
AMD_SMII_GFX_80 = """gpu,power_usage,hotspot_temperature,memory_temperature,gfx,gfx_clock,mem,mem_clock,encoder,decoder,vclock,dclock,single_bit_ecc,double_bit_ecc,pcie_replay,vram_used,vram_total,pcie_bw,pviol,tviol,phot_tviol,vr_tviol,hbm_tviol
0,180,78,82,80,1800,90,200,N/A,45.0,25,25,0,0,N/A,14000,16334,N/A,N/A,N/A,N/A,N/A,N/A
"""
AMD_SMII_JSON = _load_fixture("amd-smi-monitor--json.txt")
AMD_SMII_GFX_0 = '[{"gpu":0,"gfx":{"value":0,"unit":"%"}}]'
AMD_SMII_GFX_80 = '[{"gpu":0,"gfx":{"value":80,"unit":"%"}}]'
WHO_WITH_USERS = _load_fixture("who.with-users.txt")
WHO_EMPTY = ""
@@ -65,10 +61,6 @@ MPSTAT_HIGH = """Linux 6.17.11-200.fc42.x86_64 (gpu-01) 06/08/26 _x86_64_ (12
Average: all 85.30 0.00 8.20 0.50 0.00 0.10 0.00 0.00 0.00 5.90
"""
AMD_SMII_ONLY_HEADER = """gpu,power_usage,hotspot_temperature,memory_temperature,gfx
0,20,52,62,0
"""
# ---------------------------------------------------------------------------
# Tests
@@ -194,6 +186,22 @@ class TestMeasureCpu(unittest.TestCase):
self.assertIsNone(result)
logger.warning.assert_called_with("mpstat timed out")
def test_portuguese_locale(self):
"""Verify mpstat with C locale (LC_ALL=C) parses correctly.
The measure_cpu function forces LC_ALL=C so mpstat always outputs
in English regardless of system locale.
"""
from auto_suspend import measure_cpu
logger = MagicMock()
with patch("auto_suspend.subprocess.run") as run:
run.return_value = _make_mock_result(MPSTAT_IDLE)
result = measure_cpu(logger)
# Verify subprocess was called with LC_ALL=C
call_kwargs = run.call_args
self.assertEqual(call_kwargs[1]["env"]["LC_ALL"], "C")
self.assertAlmostEqual(result, 99.58, places=1)
class TestMeasureGpuNvidia(unittest.TestCase):
@@ -253,7 +261,7 @@ class TestMeasureGpuAmd(unittest.TestCase):
from auto_suspend import measure_gpu_amd
logger = MagicMock()
with patch("auto_suspend.subprocess.run") as run:
run.return_value = _make_mock_result(AMD_SMII_GFX)
run.return_value = _make_mock_result(AMD_SMII_JSON)
result = measure_gpu_amd(logger)
self.assertAlmostEqual(result, 100.0, places=1)
@@ -289,6 +297,14 @@ class TestMeasureGpuAmd(unittest.TestCase):
result = measure_gpu_amd(logger)
self.assertIsNone(result)
def test_empty_array(self):
from auto_suspend import measure_gpu_amd
logger = MagicMock()
with patch("auto_suspend.subprocess.run") as run:
run.return_value = _make_mock_result("[]")
result = measure_gpu_amd(logger)
self.assertIsNone(result)
def test_returncode_nonzero(self):
from auto_suspend import measure_gpu_amd
logger = MagicMock()
@@ -371,6 +387,77 @@ class TestUpdateAverages(unittest.TestCase):
self.assertAlmostEqual(state["gpu_avg"], 5.0)
self.assertEqual(state["sample_count"], 2)
def test_ema_decay_idle_gpu(self):
"""
Simulate GPU going from 100% to 0% idle and verify EMA decays.
This validates that the EMA calculation correctly handles the scenario
where a busy GPU becomes idle — the average should converge toward
idle values after enough consecutive idle samples.
"""
from auto_suspend import fresh_state, update_averages
logger = MagicMock()
state = fresh_state()
# Phase 1: GPU was busy (simulate 10 consecutive 100% samples)
for _ in range(10):
state = update_averages(state, cpu_n=95.0, gpu_n=100.0, alpha=0.3, logger=logger)
self.assertAlmostEqual(state["gpu_avg"], 100.0, places=1)
self.assertEqual(state["sample_count"], 10)
# Phase 2: GPU goes idle (0%) for 20 consecutive samples
for _ in range(20):
state = update_averages(state, cpu_n=99.0, gpu_n=0.0, alpha=0.3, logger=logger)
# After 20 idle samples from 100%, EMA should be very close to 0
# EMA = 0.7^20 * 100 ≈ 0.08%
self.assertAlmostEqual(state["gpu_avg"], 0.08, places=1)
self.assertEqual(state["sample_count"], 30)
def test_ema_convergence_multiple_values(self):
"""
Verify EMA converges to the steady-state value regardless of start.
With alpha=0.3 and 100 identical samples, the average should equal
the sample value (since EMA_new = 0.3*x + 0.7*x = x).
"""
from auto_suspend import fresh_state, update_averages
logger = MagicMock()
state = fresh_state()
gpu_idle = 2.0
for _ in range(50):
state = update_averages(state, cpu_n=95.0, gpu_n=gpu_idle, alpha=0.3, logger=logger)
self.assertAlmostEqual(state["gpu_avg"], gpu_idle, places=2)
self.assertAlmostEqual(state["cpu_avg"], 95.0, places=2)
def test_ema_with_missing_gpu_sample(self):
"""
Verify None GPU sample preserves the average.
When measure_gpu_amd returns None (e.g., tool unavailable), the
average should NOT be updated — it should keep the last value.
"""
from auto_suspend import fresh_state, update_averages
logger = MagicMock()
state = fresh_state()
# Seed with a GPU average
state = update_averages(state, cpu_n=95.0, gpu_n=10.0, alpha=0.3, logger=logger)
self.assertAlmostEqual(state["gpu_avg"], 10.0)
# Now GPU measurement fails (None) — average should be preserved
state = update_averages(state, cpu_n=95.0, gpu_n=None, alpha=0.3, logger=logger)
self.assertAlmostEqual(state["gpu_avg"], 10.0)
self.assertEqual(state["sample_count"], 2)
# After GPU returns, EMA should continue from the preserved value
state = update_averages(state, cpu_n=95.0, gpu_n=0.0, alpha=0.3, logger=logger)
# EMA = 0.3 * 0 + 0.7 * 10 = 7.0
self.assertAlmostEqual(state["gpu_avg"], 7.0)
class TestEvaluate(unittest.TestCase):