Kiwy-Signage/linux/monitor_player.py

"""monitor_player.py — CPU, GPU, temperature and memory monitor for the player.

A 24/7 signage player on a Raspberry Pi 4 fails in slow, quiet ways: the SoC
throttles, the CPU governor parks a core, memory creeps up until the OOM killer
picks a victim. This samples the numbers that reveal those trends before they
become a blank screen.

Usage:
    # live view, 5s interval, until Ctrl+C
    .venv/bin/python linux/monitor_player.py

    # 10 minutes at 2s, also written to CSV
    .venv/bin/python linux/monitor_player.py --interval 2 --duration 600

    # background logging only (no live view)
    .venv/bin/python linux/monitor_player.py --quiet --duration 3600 \
        --csv logs/monitor-$(date +%F-%H%M).csv

What is measured, and why each one matters:

| Metric | Why it matters here |
|--------|--------------------|
| Temperature | Pi 4 throttles at 80 °C (soft limit 85 °C). Sustained high temp means the case or heatsink is inadequate for 24/7. |
| Throttle flags | ``vcgencmd get_throttled`` distinguishes *currently throttled* from *has throttled since boot* — the second one only (bits 16-19) reveals a problem that already happened. |
| ARM clock | Dropping below the configured max mid-run is the signature of thermal/voltage capping. |
| V3D / pixel clock | The VideoCore GPU clock. A weblink drives it; a static image barely does. |
| Core voltage | Under-voltage (bit 0 of the throttle mask) is almost always an inadequate PSU, and it corrupts SD cards. |
| Per-core CPU | One pegged core is normal for Kivy (single-threaded main loop); all four pegged is not. |
| Player PSS | Proportional set size of the player process — the honest figure. |
| Chromium total | A *count* as much as memory: a rising count means weblinks are leaking. |
| /dev/shm | Chromium renders through shared memory; if it fills, pages render blank. |

GPU utilisation is deliberately absent: the Pi 4's V3D exposes no
``gpu_busy_percent`` (that is a Pi 5 / v3d-drm feature) and
``/sys/kernel/debug/dri/0/gpu_stats`` does not exist here. GPU activity is
therefore inferred from the clock domains and the temperature, which is the
best this hardware offers.
"""

from __future__ import annotations

import argparse
import csv
import os
import subprocess
import sys
import time
from datetime import datetime

try:
    import psutil
except ImportError:
    print('psutil is required: .venv/bin/pip install psutil', file=sys.stderr)
    raise SystemExit(2)


# ── Constants ────────────────────────────────────────────────────────
#: Pi 4 begins soft-throttling around here. Warning threshold, not a limit.
TEMP_WARN_C = 70.0
#: Active soft-throttle limit on Pi 4.
TEMP_CRITICAL_C = 80.0

#: ``vcgencmd get_throttled`` bit meanings. The 16-19 group is sticky: it
#: records that the condition occurred at any point since boot, which is the
#: only way to catch an intermittent brown-out on an unattended device.
THROTTLE_BITS = {
    0: ('NOW', 'under-voltage'),
    1: ('NOW', 'arm frequency capped'),
    2: ('NOW', 'currently throttled'),
    3: ('NOW', 'soft temperature limit'),
    16: ('HAS', 'under-voltage occurred'),
    17: ('HAS', 'arm frequency capping occurred'),
    18: ('HAS', 'throttling occurred'),
    19: ('HAS', 'soft temperature limit occurred'),
}

CLOCK_DOMAINS = ('arm', 'core', 'v3d', 'h264', 'pixel')

CSV_FIELDS = [
    'timestamp', 'uptime_s',
    'cpu_total_pct', 'cpu0_pct', 'cpu1_pct', 'cpu2_pct', 'cpu3_pct',
    'load1', 'load5', 'load15',
    'temp_c', 'volt_core_v',
    'arm_mhz', 'core_mhz', 'v3d_mhz', 'pixel_mhz', 'h264_mhz',
    'governor', 'arm_cur_mhz_cfg',
    'mem_used_mb', 'mem_avail_mb', 'shm_used_mb',
    'player_pss_mb', 'player_cpu_pct', 'player_threads',
    'chromium_procs', 'chromium_pss_mb',
    'throttled_raw', 'throttle_flags',
]


def _run(args, timeout=5):
    """Run a command, return stripped stdout, or '' on any failure."""
    try:
        result = subprocess.run(
            args, capture_output=True, text=True, timeout=timeout, check=False,
        )
        return (result.stdout or '').strip()
    except Exception:
        return ''


def _read(path):
    try:
        with open(path) as fh:
            return fh.read().strip()
    except Exception:
        return ''


# ── Individual metrics ───────────────────────────────────────────────
def read_temp_c():
    """SoC temperature in °C from hwmon, falling back to vcgencmd/thermal."""
    raw = _read('/sys/class/hwmon/hwmon0/temp1_input')
    if raw.isdigit():
        return int(raw) / 1000.0
    for zone in ('/sys/class/thermal/thermal_zone0/temp',):
        raw = _read(zone)
        if raw.isdigit():
            return int(raw) / 1000.0
    out = _run(['vcgencmd', 'measure_temp'])
    # "temp=50.1'C"
    try:
        return float(out.split('=')[1].split("'")[0])
    except Exception:
        return None


def read_clock_mhz(domain):
    """Clock frequency in MHz for a vcgencmd domain."""
    out = _run(['vcgencmd', 'measure_clock', domain])
    # "frequency(48)=1800457088"
    try:
        return int(out.split('=')[1]) / 1_000_000.0
    except Exception:
        return None


def read_core_volts():
    out = _run(['vcgencmd', 'measure_volts', 'core'])
    # "volt=0.9160V"
    try:
        return float(out.split('=')[1].rstrip('V'))
    except Exception:
        return None


def read_throttled():
    """(raw_int, [human readable flags]) from vcgencmd get_throttled."""
    out = _run(['vcgencmd', 'get_throttled'])
    # "throttled=0x0"
    try:
        raw = int(out.split('=')[1], 16)
    except Exception:
        return None, []
    flags = []
    for bit, (when, label) in THROTTLE_BITS.items():
        if raw & (1 << bit):
            flags.append(f'{when}:{label}')
    return raw, flags


def pss_mb(pid):
    """PSS in MB for a process, or None. RSS double-counts shared pages."""
    total_kb = 0
    try:
        with open(f'/proc/{pid}/smaps_rollup') as fh:
            for line in fh:
                if line.startswith('Pss:'):
                    total_kb = int(line.split()[1])
                    break
    except Exception:
        return None
    return total_kb / 1024.0


def find_procs(pattern):
    """PIDs whose cmdline matches ``pattern``, excluding this script."""
    pids = []
    me = os.getpid()
    for proc in psutil.process_iter(['pid', 'cmdline']):
        try:
            if proc.info['pid'] == me:
                continue
            cmdline = ' '.join(proc.info['cmdline'] or [])
            if pattern in cmdline and 'monitor_player' not in cmdline:
                pids.append(proc.info['pid'])
        except Exception:
            continue
    return pids


#: ``psutil.Process`` objects kept alive between samples. ``cpu_percent()``
#: needs a *previous* reading for the same process to compute a delta, so a
#: freshly constructed Process always reports 0.0. Caching these is what makes
#: the player's own CPU figure meaningful instead of a constant zero.
_proc_cache: dict[int, 'psutil.Process'] = {}


def player_cpu_percent(pid):
    """CPU % for ``pid`` since the previous sample."""
    try:
        proc = _proc_cache.get(pid)
        if proc is None or not proc.is_running():
            proc = psutil.Process(pid)
            _proc_cache[pid] = proc
            proc.cpu_percent(interval=None)  # establish the baseline
            return 0.0
        return proc.cpu_percent(interval=None)
    except Exception:
        return None


def sample():
    """Take one measurement. Never raises; missing metrics become None."""
    now = time.time()
    row = {'timestamp': datetime.now().isoformat(timespec='seconds')}

    # CPU. interval=None means "since the previous call" — with a fixed sample
    # period this yields the usage over exactly that window.
    try:
        row['cpu_total_pct'] = psutil.cpu_percent(interval=None)
        per_core = psutil.cpu_percent(interval=None, percpu=True)
        for i in range(4):
            row[f'cpu{i}_pct'] = per_core[i] if i < len(per_core) else None
        load = os.getloadavg()
        row['load1'], row['load5'], row['load15'] = load
    except Exception:
        row.update({'cpu_total_pct': None, 'load1': None, 'load5': None,
                    'load15': None})

    # Thermal / power
    row['temp_c'] = read_temp_c()
    row['volt_core_v'] = read_core_volts()

    # Clocks
    for domain in CLOCK_DOMAINS:
        row[f'{domain}_mhz'] = read_clock_mhz(domain)

    # Governor and configured max
    row['governor'] = _read(
        '/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor')
    cur = _read('/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq')
    row['arm_cur_mhz_cfg'] = int(cur) / 1000.0 if cur.isdigit() else None

    # Memory
    try:
        mem = psutil.virtual_memory()
        row['mem_used_mb'] = (mem.total - mem.available) / 1024 / 1024
        row['mem_avail_mb'] = mem.available / 1024 / 1024
        shm = psutil.disk_usage('/dev/shm')
        row['shm_used_mb'] = shm.used / 1024 / 1024
    except Exception:
        row.update({'mem_used_mb': None, 'mem_avail_mb': None,
                    'shm_used_mb': None})

    # The player
    player_pids = find_procs('run_linux.py')
    if player_pids:
        pid = player_pids[0]
        row['player_pss_mb'] = pss_mb(pid)
        row['player_cpu_pct'] = player_cpu_percent(pid)
        try:
            proc = _proc_cache.get(pid) or psutil.Process(pid)
            with proc.oneshot():
                row['player_threads'] = proc.num_threads()
                # Uptime from the process, for correlating with playback.
                row['uptime_s'] = now - proc.create_time()
        except Exception:
            pass
    else:
        row['player_pss_mb'] = None
        row['player_cpu_pct'] = None
        row['player_threads'] = None

    # Chromium tree (weblinks). Count is as important as memory: a rising
    # count means teardown is leaking, which is what kills a 24/7 install.
    chromium_pids = find_procs('chromium')
    row['chromium_procs'] = len(chromium_pids)
    total = 0.0
    for pid in chromium_pids:
        value = pss_mb(pid)
        if value:
            total += value
    row['chromium_pss_mb'] = total if chromium_pids else 0.0

    # Throttling
    raw, flags = read_throttled()
    row['throttled_raw'] = hex(raw) if raw is not None else None
    row['throttle_flags'] = ','.join(flags) if flags else ''

    # Fill any field we never set so the CSV stays rectangular.
    for field in CSV_FIELDS:
        row.setdefault(field, None)
    return row


# ── Presentation ─────────────────────────────────────────────────────
def fmt(value, width=6, spec='.1f'):
    if value is None:
        return ' ' * (width - 2) + '--'
    return f'{value:{width}{spec}}'


def print_header():
    print()
    print('   time    CPU%  cores(0-3)   temp    ARM    V3D  volt   RAM  '
          'player chrome  gov      flags')
    print('  ' + '-' * 84)


def print_row(row, tick):
    cores = ' '.join(
        fmt(row.get(f'cpu{i}_pct'), 3, '.0f') for i in range(4))
    flags = (row.get('throttle_flags') or '').replace('HAS:', 'had:') \
        .replace('NOW:', 'NOW:')
    warn = ''
    temp = row.get('temp_c')
    if temp is not None and temp >= TEMP_CRITICAL_C:
        warn = ' ***HOT***'
    elif temp is not None and temp >= TEMP_WARN_C:
        warn = ' *warm*'

    governor = (row.get('governor') or '')[:6]
    print(
        f'  {row["timestamp"][11:19]} '
        f'{fmt(row.get("cpu_total_pct"), 4, ".0f")} '
        f' {cores}  '
        f'{fmt(temp, 5)}C '
        f'{fmt(row.get("arm_mhz"), 6, ".0f")} '
        f'{fmt(row.get("v3d_mhz"), 5, ".0f")} '
        f'{fmt(row.get("volt_core_v"), 6, ".3f")}V'
        f'{fmt(row.get("mem_used_mb"), 5, ".0f")}M '
        f'{fmt(row.get("player_pss_mb"), 4, ".0f")}M '
        f'{fmt(row.get("chromium_pss_mb"), 5, ".0f")}M '
        f'{governor:<6} '
        f'{flags}{warn}'
    )


def summarise(rows):
    """Print min/avg/max for the run — the part that reveals a trend."""
    if not rows:
        return
    print('\n  ' + '=' * 84)
    print(f'  SUMMARY over {len(rows)} samples '
          f'({rows[0]["timestamp"][11:19]} -> {rows[-1]["timestamp"][11:19]})')
    print('  ' + '=' * 84)

    def stat(field, spec='.1f', unit=''):
        values = [r[field] for r in rows
                  if isinstance(r.get(field), (int, float))]
        if not values:
            return f'  {field:<18} (no data)'
        return (f'  {field:<18} min {min(values):>8{spec}}{unit}   '
                f'avg {sum(values)/len(values):>8{spec}}{unit}   '
                f'max {max(values):>8{spec}}{unit}')

    for field, spec, unit in (
        ('cpu_total_pct', '.1f', '%'),
        ('temp_c', '.1f', 'C'),
        ('arm_mhz', '.0f', 'MHz'),
        ('v3d_mhz', '.0f', 'MHz'),
        ('volt_core_v', '.3f', 'V'),
        ('mem_avail_mb', '.0f', 'MB'),
        ('player_pss_mb', '.0f', 'MB'),
        ('player_cpu_pct', '.1f', '%'),
        ('chromium_procs', '.0f', ''),
        ('chromium_pss_mb', '.0f', 'MB'),
    ):
        print(stat(field, spec, unit))

    # Trend: first vs last quarter, which is what matters for a leak.
    def trend(field):
        values = [r[field] for r in rows
                  if isinstance(r.get(field), (int, float))]
        if len(values) < 8:
            return ''
        head = values[:max(1, len(values) // 4)]
        tail = values[-max(1, len(values) // 4):]
        delta = sum(tail) / len(tail) - sum(head) / len(head)
        arrow = 'RISING ' if delta > 0 else 'falling'
        return (f'  {field:<18} {arrow}  {delta:+8.1f} '
                f'(first->last quarter)')

    print('\n  Trend (the leak check):')
    for field in ('player_pss_mb', 'chromium_procs', 'mem_avail_mb', 'temp_c'):
        line = trend(field)
        if line:
            print(line)

    # Throttle history is the headline finding on a Pi.
    all_flags = set()
    for row in rows:
        if row.get('throttle_flags'):
            all_flags.update(row['throttle_flags'].split(','))
    print('\n  Throttle / power events during this run:')
    print(f'    {", ".join(sorted(all_flags)) if all_flags else "none — clean"}')


def main():
    parser = argparse.ArgumentParser(
        description='Monitor CPU, GPU clock, temperature and memory for the player.')
    parser.add_argument('--interval', type=float, default=5.0,
                        help='seconds between samples (default 5)')
    parser.add_argument('--duration', type=float, default=0,
                        help='seconds to run; 0 = until Ctrl+C (default 0)')
    parser.add_argument('--csv', default='',
                        help='write samples to this CSV file')
    parser.add_argument('--quiet', action='store_true',
                        help='no live output (for background logging)')
    args = parser.parse_args()

    # Prime the CPU counters: the first cpu_percent() call always returns 0.0
    # because it has no previous sample to compare against.
    psutil.cpu_percent(interval=None)
    psutil.cpu_percent(interval=None, percpu=True)
    for pid in find_procs('run_linux.py'):
        try:
            _proc_cache[pid] = psutil.Process(pid)
            _proc_cache[pid].cpu_percent(interval=None)
        except Exception:
            pass

    writer = None
    csv_handle = None
    if args.csv:
        os.makedirs(os.path.dirname(os.path.abspath(args.csv)), exist_ok=True)
        csv_handle = open(args.csv, 'w', newline='')
        writer = csv.DictWriter(csv_handle, fieldnames=CSV_FIELDS)
        writer.writeheader()

    if not args.quiet:
        print_header()

    rows = []
    started = time.time()
    tick = 0
    try:
        while True:
            time.sleep(args.interval)
            tick += 1
            row = sample()
            rows.append(row)
            if writer:
                writer.writerow(row)
                csv_handle.flush()
            if not args.quiet:
                print_row(row, tick)
            if args.duration and (time.time() - started) >= args.duration:
                break
    except KeyboardInterrupt:
        if not args.quiet:
            print('\n  stopped by user')
    finally:
        if csv_handle:
            csv_handle.close()
            if not args.quiet:
                print(f'  CSV written to {args.csv}')
        if not args.quiet:
            summarise(rows)
    return 0


if __name__ == '__main__':
    raise SystemExit(main())