alexpires.me/a13labs.infra
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

auto_suspend: binary GPU idle counter replaces EMA for GPU

Browse Source
This commit is contained in:
Alexandre Pires
2026-06-09 22:13:45 +02:00
parent 4825ad1495
commit df5a98853b
2 changed files with 186 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ansible/roles/auto_suspend/files/auto_suspend.py
+47 -40
View File
@@ -2,9 +2,10 @@
"""
Auto-suspend daemon — checks system idleness and suspends when idle.
Uses an exponential moving average (EMA) of CPU idle % and GPU utilization
to smooth out transient spikes. The system suspends only after
idle_timeout_cycles consecutive idle checks.
Uses an exponential moving average (EMA) of CPU idle % to smooth out
transient spikes. GPU utilization is tracked as a binary busy/idle value
per cycle: if any GPU exceeds the threshold, the idle counter resets.
The system suspends only after idle_timeout_cycles consecutive idle checks.
State is persisted to /var/lib/auto_suspend/state.json and survives
across timer invocations. State is fully reset on suspend resume.
@@ -78,10 +79,8 @@ def fresh_state():
return {
"cpu_idle_n": None,
"cpu_idle_n1": None,
"gpu_util_n": None,
"gpu_util_n1": None,
"cpu_avg": None,
"gpu_avg": None,
"gpu_idle_counter": 0,
"idle_counter": 0,
"sample_count": 0,
}
@@ -101,8 +100,9 @@ def load_state(state_dir, logger):
for key in fresh_state():
if key not in state:
state[key] = fresh_state()[key]
logger.debug("Loaded state: sample_count=%d, idle_counter=%d",
state["sample_count"], state["idle_counter"])
logger.debug("Loaded state: sample_count=%d, idle_counter=%d, gpu_idle=%d",
state["sample_count"], state["idle_counter"],
state.get("gpu_idle_counter", 0))
return state
except (json.JSONDecodeError, IOError, OSError) as exc:
logger.warning("Failed to load state (%s) — starting fresh", exc)
@@ -122,8 +122,9 @@ def save_state(state_dir, state, logger):
json.dump(state, f, indent=2)
f.write("\n")
os.replace(tmp, str(state_path))
logger.debug("State saved: sample_count=%d, idle_counter=%d",
state["sample_count"], state["idle_counter"])
logger.debug("State saved: sample_count=%d, idle_counter=%d, gpu_idle=%d",
state["sample_count"], state["idle_counter"],
state.get("gpu_idle_counter", 0))
except BaseException:
try:
os.unlink(tmp)
@@ -288,31 +289,33 @@ def get_active_users(logger):
# EMA update
# ---------------------------------------------------------------------------
def update_averages(state, cpu_n, gpu_n, alpha, logger):
def update_averages(state, cpu_n, gpu_n, gpu_threshold, alpha, logger):
"""
Update EMA averages using the new measurement and shift n → n-1.
Update CPU EMA averages and GPU idle counter.
GPU is binary per-cycle: if utilization > threshold, the GPU idle
counter resets. If idle (below threshold), the counter increments.
Args:
state: current state dict (modified in place for averages)
cpu_n: current CPU idle %
gpu_n: current GPU utilization %
gpu_n: current GPU utilization % (None = no GPU, treated as idle)
gpu_threshold: utilization threshold above which GPU is "busy"
alpha: EMA weight for current sample
logger: logger instance
Returns:
dict with updated state (new keys: cpu_avg, gpu_avg, cpu_idle_n,
gpu_util_n, cpu_idle_n1, gpu_util_n1, sample_count)
dict with updated state (new keys: cpu_avg, cpu_idle_n, cpu_idle_n1,
gpu_idle_counter, sample_count)
"""
new_state = dict(state)
new_state["sample_count"] = state["sample_count"] + 1
# Shift current → previous
new_state["cpu_idle_n1"] = state["cpu_idle_n"]
new_state["gpu_util_n1"] = state["gpu_util_n"]
# Update current measurements
new_state["cpu_idle_n"] = cpu_n
new_state["gpu_util_n"] = gpu_n
# EMA for CPU idle average
if state["cpu_avg"] is not None and cpu_n is not None:
@@ -321,16 +324,20 @@ def update_averages(state, cpu_n, gpu_n, alpha, logger):
# First valid sample seeds the average
new_state["cpu_avg"] = cpu_n
# EMA for GPU utilization average
if state["gpu_avg"] is not None and gpu_n is not None:
new_state["gpu_avg"] = round(alpha * gpu_n + (1 - alpha) * state["gpu_avg"], 2)
elif gpu_n is not None:
new_state["gpu_avg"] = gpu_n
# GPU idle counter — binary per-cycle (no EMA)
gpu_busy = gpu_n is not None and gpu_n > gpu_threshold
if gpu_busy:
new_state["gpu_idle_counter"] = 0
logger.debug("GPU busy (%.1f%% >= %.0f%%) — reset gpu_idle_counter",
gpu_n, gpu_threshold)
else:
# gpu_n is None (no GPU) or below threshold — counts as idle
new_state["gpu_idle_counter"] = state.get("gpu_idle_counter", 0) + 1
logger.debug(
"Updated averages: cpu_avg=%.1f%%, gpu_avg=%.1f%%, sample_count=%d",
"Updated averages: cpu_avg=%.1f%%, gpu_idle_counter=%d, sample_count=%d",
new_state["cpu_avg"] if new_state["cpu_avg"] is not None else 0,
new_state["gpu_avg"] if new_state["gpu_avg"] is not None else 0,
new_state["gpu_idle_counter"],
new_state["sample_count"],
)
return new_state
@@ -340,7 +347,8 @@ def update_averages(state, cpu_n, gpu_n, alpha, logger):
# ---------------------------------------------------------------------------
def get_why_not_idle(state, has_users, cpu_idle_threshold=DEFAULT_CPU_IDLE_THRESHOLD,
gpu_util_threshold=DEFAULT_GPU_UTIL_THRESHOLD, min_samples=DEFAULT_MIN_SAMPLES, logger=None):
gpu_util_threshold=DEFAULT_GPU_UTIL_THRESHOLD, min_samples=DEFAULT_MIN_SAMPLES,
gpu_idle_counter=None, logger=None):
"""Return a human-readable reason why the system is not idle."""
reasons = []
if has_users:
@@ -350,11 +358,8 @@ def get_why_not_idle(state, has_users, cpu_idle_threshold=DEFAULT_CPU_IDLE_THRES
reasons.append(
f"CPU idle avg {state['cpu_avg']:.1f}% <= threshold {cpu_idle_threshold:.0f}%"
)
if state["gpu_avg"] is not None:
if state["gpu_avg"] >= gpu_util_threshold:
reasons.append(
f"GPU util avg {state['gpu_avg']:.1f}% >= threshold {gpu_util_threshold:.0f}%"
)
if gpu_idle_counter is not None and gpu_idle_counter == 0 and state["sample_count"] >= min_samples:
reasons.append("GPU busy (idle_counter=0)")
if not reasons:
if state["sample_count"] < min_samples:
reasons.append(f"warming up ({state['sample_count']}/{min_samples} samples)")
@@ -363,11 +368,12 @@ def get_why_not_idle(state, has_users, cpu_idle_threshold=DEFAULT_CPU_IDLE_THRES
return "; ".join(reasons)
def evaluate(state, has_users, idle_threshold, gpu_threshold,
def evaluate(state, has_users, gpu_idle_counter, idle_threshold, gpu_threshold,
min_samples, logger):
"""
Evaluate whether the system should be considered idle.
GPU is binary per-cycle: busy if any GPU > threshold, otherwise idle.
Returns True if all conditions are met for one idle cycle.
"""
# Need enough samples before trusting the average
@@ -391,11 +397,10 @@ def evaluate(state, has_users, idle_threshold, gpu_threshold,
)
return False
# Check GPU utilization average (None means no GPUs = ok)
if state["gpu_avg"] is not None and state["gpu_avg"] >= gpu_threshold:
# GPU is binary per-cycle: busy if counter was reset this cycle
if gpu_idle_counter == 0 and state["sample_count"] >= min_samples:
logger.debug(
"Not idle: GPU avg %.1f%% >= threshold %.0f%%",
state["gpu_avg"], gpu_threshold,
"Not idle: GPU busy this cycle (idle_counter=0)"
)
return False
@@ -457,14 +462,14 @@ def main():
# --- Update averages ---
state = update_averages(
state, cpu_idle, gpu_util, args.alpha, logger,
state, cpu_idle, gpu_util, args.gpu_util_threshold, args.alpha, logger,
)
# --- Evaluate ---
has_users = get_active_users(logger)
is_idle = evaluate(
state, has_users,
state, has_users, state["gpu_idle_counter"],
idle_threshold=args.cpu_idle_threshold,
gpu_threshold=args.gpu_util_threshold,
min_samples=args.min_samples,
@@ -474,10 +479,10 @@ def main():
if is_idle:
state["idle_counter"] = state.get("idle_counter", 0) + 1
logger.info(
"System idle (cpu_avg=%.1f%%, gpu_avg=%.1f%%, "
"System idle (cpu_avg=%.1f%%, gpu_idle=%d, "
"idle_counter=%d/%d)",
state["cpu_avg"] or 0,
state["gpu_avg"] or 0,
state["gpu_idle_counter"],
state["idle_counter"],
args.idle_timeout_cycles,
)
@@ -512,7 +517,9 @@ def main():
else:
if not args.dry_run:
state["idle_counter"] = 0
reason = get_why_not_idle(state, has_users, args.cpu_idle_threshold, args.gpu_util_threshold, args.min_samples, logger)
reason = get_why_not_idle(
state, has_users, args.cpu_idle_threshold, args.gpu_util_threshold,
args.min_samples, state.get("gpu_idle_counter"), logger)
logger.info("Not idle (%s) — reset counter", reason)
# --- Save and exit ---