<# ===================================================================== watchdog.ps1 - keep the Kiwy Signage Player running 24/7 on Windows ===================================================================== WHAT THIS DOES Supervises KiwySignagePlayer.exe and restarts it when it: * CRASHES - the process disappeared without the user asking it to. * HANGS - the process is alive but its heartbeat file has gone stale, which means the UI thread is wedged (a frozen player looks exactly like a working one from the outside). This is the Windows counterpart of the Linux `start.sh` watchdog, which has been running the player on the Raspberry Pi deployments. HOW THE "PASSWORD ONLY" EXIT IS PRESERVED The only supported way to stop the player is the exit screen's password. On success the player writes a stop-flag file next to its .exe: .player_stop_requested The flag is SESSION SCOPED: * While the flag exists, this watchdog will NOT restart the player; it stops supervising and exits (the machine is left with the player off). * Every time the watchdog starts, it CLEARS the flag first, which is what makes a fresh launch a fresh session. So starting the player again is all it takes to resume - there is no file to delete by hand. CRASH-LOOP BREAKER A player that cannot stay up (bad config, missing media, GPU problem) would otherwise be restarted forever. If the player dies `CrashLoopMaxFailures` times inside `CrashLoopWindowMin` minutes WITHOUT ever becoming healthy, the watchdog backs off for `CrashLoopBackoffMin` minutes before retrying, and keeps the reason in the log. WHAT IT DOES NOT DO It does not install itself to run at boot/login - start it with `start_player_watchdog.bat`. It also cannot watch the player before the player has ever run, so a totally broken install simply logs and backs off. USAGE powershell -NoProfile -ExecutionPolicy Bypass -File watchdog.ps1 powershell -NoProfile -ExecutionPolicy Bypass -File watchdog.ps1 -Status Stop it with Ctrl+C in its window (this does NOT stop the player; it only stops supervising). #> [CmdletBinding()] param( # Path to the player executable. Defaults to the standard build output. [string]$ExePath, # How often the supervisor checks on the player. [int]$HealthCheckIntervalSec = 15, # A heartbeat older than this means the player is wedged, since the player # rewrites it every 10 seconds. Matches the proven Linux value of 60s. [int]$HeartbeatStaleSec = 60, # After launching, give the player this long to write its first heartbeat # (startup shows a splash video, so it is not instant) before health # checks count against it. [int]$StartupGraceSec = 120, # Pause before restarting a player that died. [int]$RestartDelaySec = 5, # Crash-loop breaker (see header). [int]$CrashLoopWindowMin = 10, [int]$CrashLoopMaxFailures = 5, [int]$CrashLoopBackoffMin = 10, # Report current state and exit - never launches or kills anything. [switch]$Status ) $ErrorActionPreference = 'Stop' # UTF-8 so accented paths in logs are readable. $OutputEncoding = [System.Text.Encoding]::UTF8 # --------------------------------------------------------------------- # Paths # --------------------------------------------------------------------- function Resolve-ExePath { param([string]$Override) if ($Override) { if (-not (Test-Path -LiteralPath $Override)) { throw "Player executable not found: $Override" } return (Resolve-Path -LiteralPath $Override).Path } $candidates = @( (Join-Path $PSScriptRoot 'dist\KiwySignagePlayer\KiwySignagePlayer.exe'), (Join-Path $PSScriptRoot 'KiwySignagePlayer.exe') ) foreach ($c in $candidates) { if (Test-Path -LiteralPath $c) { return (Resolve-Path -LiteralPath $c).Path } } throw ("Could not find KiwySignagePlayer.exe. Looked in:`n " + ($candidates -join "`n ")) } $ExeFull = Resolve-ExePath -Override $ExePath $PlayerDir = Split-Path -Parent $ExeFull $HeartbeatFile = Join-Path $PlayerDir '.player_heartbeat' $StopFlagFile = Join-Path $PlayerDir '.player_stop_requested' $LogDir = Join-Path $PlayerDir 'logs' $LogFile = Join-Path $LogDir 'watchdog.log' $ProcessName = [System.IO.Path]::GetFileNameWithoutExtension($ExeFull) New-Item -ItemType Directory -Force -Path $LogDir | Out-Null # --------------------------------------------------------------------- # Logging (kept tiny and never throws) # --------------------------------------------------------------------- function Write-Log { param( [Parameter(Mandatory = $true)][string]$Message, [ValidateSet('INFO', 'WARN', 'ERROR')][string]$Level = 'INFO' ) $line = "[{0}] [{1}] {2}" -f (Get-Date -Format 'yyyy-MM-dd HH:mm:ss'), $Level, $Message Write-Host $line try { Add-Content -LiteralPath $LogFile -Value $line -Encoding UTF8 } catch { # Logging must never take the supervisor down. } } # --------------------------------------------------------------------- # Player process helpers # --------------------------------------------------------------------- function Get-PlayerProcesses { # NOTE: the packaged player is TWO processes on Windows - the PyInstaller # bootloader parent and the child that owns the SDL window. Both share the # image name, so selecting by name covers the whole tree. @(Get-Process -Name $ProcessName -ErrorAction SilentlyContinue) } function Get-PlayerRootPid { $procs = Get-PlayerProcesses if ($procs.Count -eq 0) { return $null } # Prefer the child (it has the main window); fall back to any. $withWindow = $procs | Where-Object { $_.MainWindowHandle -ne 0 } | Select-Object -First 1 if ($withWindow) { return $withWindow.Id } return ($procs | Select-Object -First 1).Id } function Test-HeartbeatFresh { param([int]$MaxAgeSec) if (-not (Test-Path -LiteralPath $HeartbeatFile)) { return $false } try { $age = (Get-Date).ToUniversalTime() - ` (Get-Item -LiteralPath $HeartbeatFile).LastWriteTimeUtc return ($age.TotalSeconds -lt $MaxAgeSec) } catch { return $false } } function Get-HeartbeatAgeSec { if (-not (Test-Path -LiteralPath $HeartbeatFile)) { return -1 } try { $age = (Get-Date).ToUniversalTime() - ` (Get-Item -LiteralPath $HeartbeatFile).LastWriteTimeUtc return [int]$age.TotalSeconds } catch { return -1 } } function Stop-PlayerTree { param([int]$RootPid) # /T kills the PyInstaller child too; without it the visible player window # would survive and the next launch would collide with it. try { & taskkill.exe /F /T /PID $RootPid 2>&1 | Out-Null } catch { Write-Log "taskkill failed for pid $RootPid ($($_.Exception.Message)); forcing by name" 'WARN' } # Belt and braces: make sure no stragglers remain. $deadline = (Get-Date).AddSeconds(15) while ((Get-Date) -lt $deadline) { if ((Get-PlayerProcesses).Count -eq 0) { return $true } Start-Sleep -Milliseconds 500 } foreach ($p in Get-PlayerProcesses) { try { Stop-Process -Id $p.Id -Force -ErrorAction SilentlyContinue } catch { } } Start-Sleep -Milliseconds 500 return ((Get-PlayerProcesses).Count -eq 0) } function Start-Player { Write-Log "Launching player: $ExeFull" # Start in the player's own folder: the app resolves config/media/playlists # relative to its working directory. Start-Process -FilePath $ExeFull -WorkingDirectory $PlayerDir | Out-Null } # --------------------------------------------------------------------- # -Status : report and exit (never mutates anything) # --------------------------------------------------------------------- if ($Status) { $procs = Get-PlayerProcesses $pid0 = Get-PlayerRootPid $age = Get-HeartbeatAgeSec Write-Host '==========================================' Write-Host ' Kiwy Signage Player - Watchdog Status' Write-Host '==========================================' Write-Host ("Executable : {0}" -f $ExeFull) Write-Host ("Processes : {0}" -f $procs.Count) if ($pid0) { Write-Host ("Root PID : {0}" -f $pid0) } if ($age -ge 0) { Write-Host ("Heartbeat : {0}s old" -f $age) if ($age -lt $HeartbeatStaleSec) { Write-Host 'Health : HEALTHY' -ForegroundColor Green } else { Write-Host 'Health : STALE (player may be hung)' -ForegroundColor Yellow } } else { Write-Host 'Heartbeat : (not present - player has not started)' } if (Test-Path -LiteralPath $StopFlagFile) { Write-Host 'Stop flag : PRESENT (password exit was used)' Write-Host ' a new launch clears it' -ForegroundColor Yellow } else { Write-Host 'Stop flag : absent (watchdog would restart on failure)' } Write-Host ("Log : {0}" -f $LogFile) exit 0 } # --------------------------------------------------------------------- # Session start: clear the stop flag # --------------------------------------------------------------------- # This is what makes the flag session-scoped: the user's password exit ends # the CURRENT session, and the next launch begins a new one. if (Test-Path -LiteralPath $StopFlagFile) { try { Remove-Item -LiteralPath $StopFlagFile -Force Write-Log 'Cleared the stop flag from the previous session - new session started' } catch { Write-Log "Could not clear the stop flag ($($_.Exception.Message))" 'WARN' } } Write-Log '==================================================' Write-Log "Watchdog starting (exe=$ProcessName, check=${HealthCheckIntervalSec}s, stale=${HeartbeatStaleSec}s)" Write-Log "Crash-loop breaker: ${CrashLoopMaxFailures} failures / ${CrashLoopWindowMin} min -> ${CrashLoopBackoffMin} min backoff" Write-Log 'Stop the WATCHDOG with Ctrl+C. Stop the PLAYER via the exit password.' Write-Log '==================================================' # --------------------------------------------------------------------- # Crash-loop breaker # --------------------------------------------------------------------- # A player that cannot stay up (broken config, missing media, GPU fault) would # otherwise be restarted forever, filling the log and thrashing the machine. # If it fails CrashLoopMaxFailures times inside CrashLoopWindowMin minutes # WITHOUT ever becoming healthy, wait CrashLoopBackoffMin minutes before the # next attempt. Returns $true when a backoff was performed. function Invoke-CrashLoopBackoff { param([System.Collections.ArrayList]$Failures) if ($Failures.Count -lt $CrashLoopMaxFailures) { return $false } $windowStart = (Get-Date).AddMinutes(-$CrashLoopWindowMin) $recent = @($Failures | Where-Object { $_ -gt $windowStart }) if ($recent.Count -lt $CrashLoopMaxFailures) { # Only old failures remain; forget them so they cannot accumulate. $Failures.Clear() return $false } Write-Log ("Player failed $($recent.Count) times in the last " + "${CrashLoopWindowMin} minutes without ever becoming healthy.") 'ERROR' Write-Log "Backing off for ${CrashLoopBackoffMin} minutes before the next attempt." 'ERROR' Write-Log "Investigate $LogDir (and the player's own logs next to it)." 'ERROR' $backoffEnd = (Get-Date).AddMinutes($CrashLoopBackoffMin) while ((Get-Date) -lt $backoffEnd) { Start-Sleep -Seconds 5 if (Test-Path -LiteralPath $StopFlagFile) { Write-Log 'Stop flag appeared during backoff - standing down.' $Failures.Clear() return $true } } $Failures.Clear() Write-Log 'Backoff finished - resuming supervision.' return $true } # --------------------------------------------------------------------- # Main supervision loop # --------------------------------------------------------------------- # Two DIFFERENT failures need two different reactions, and confusing them # would be harmful: # # * "never became healthy" - a slow start-up, a bad install, or the # first-run setup screen. Killing on this would restart forever and could # interrupt an operator entering settings. Handled by the crash-loop # breaker, with a long hard cap before we intervene. # * "was healthy, then went silent" - the app is wedged. This is the case # that must be restarted promptly. # # So a hang is only declared once we have actually SEEN a fresh heartbeat. $failureTimestamps = New-Object System.Collections.ArrayList $hasBeenHealthy = $false while ($true) { # ---- The operator asked to exit: stand down ---------------------- if (Test-Path -LiteralPath $StopFlagFile) { Write-Log 'Stop flag present - the operator exited with the password.' Write-Log 'Watchdog will NOT restart the player. Start it again to resume.' break } $procs = Get-PlayerProcesses # ================================================================ # Case 1: the player is not running # ================================================================ if ($procs.Count -eq 0) { Write-Log 'Player is not running - starting it.' try { Start-Player } catch { Write-Log "Failed to launch the player: $($_.Exception.Message)" 'ERROR' [void]$failureTimestamps.Add((Get-Date)) [void](Invoke-CrashLoopBackoff -Failures $failureTimestamps) Start-Sleep -Seconds $RestartDelaySec continue } $launchTime = Get-Date $hasBeenHealthy = $false # Watch the start-up window: break early the moment it is healthy, # and notice immediately if it dies. $diedEarly = $false $graceEnd = $launchTime.AddSeconds($StartupGraceSec) while ((Get-Date) -lt $graceEnd) { Start-Sleep -Seconds 2 if (Test-Path -LiteralPath $StopFlagFile) { break } if ((Get-PlayerProcesses).Count -eq 0) { $diedEarly = $true; break } if (Test-HeartbeatFresh -MaxAgeSec $HeartbeatStaleSec) { $hasBeenHealthy = $true break } } if ($diedEarly -and -not (Test-Path -LiteralPath $StopFlagFile)) { [void]$failureTimestamps.Add((Get-Date)) Write-Log ("Player exited during start-up (within the ${StartupGraceSec}s grace window).") 'WARN' [void](Invoke-CrashLoopBackoff -Failures $failureTimestamps) Write-Log ("Waiting ${RestartDelaySec}s before retrying...") Start-Sleep -Seconds $RestartDelaySec } elseif ($hasBeenHealthy) { Write-Log 'Player started and is reporting a fresh heartbeat.' $failureTimestamps.Clear() } else { Write-Log 'Player is running but has not reported a heartbeat yet - continuing to watch.' 'WARN' } continue } # ================================================================ # Case 2: the player is running - is it actually working? # ================================================================ if (Test-HeartbeatFresh -MaxAgeSec $HeartbeatStaleSec) { # Healthy. Forget past failures - only CONSECUTIVE ones matter. if (-not $hasBeenHealthy -or $failureTimestamps.Count -gt 0) { Write-Log 'Player is healthy.' } $hasBeenHealthy = $true $failureTimestamps.Clear() Start-Sleep -Seconds $HealthCheckIntervalSec continue } $age = Get-HeartbeatAgeSec $ageText = if ($age -lt 0) { 'no heartbeat file' } else { "${age}s old" } if ($hasBeenHealthy) { # ---- The real hang: it WAS working and has gone silent ---------- Write-Log ("Player was healthy but its heartbeat is now stale ({0}, limit ${HeartbeatStaleSec}s) - it is hung." -f $ageText) 'ERROR' $rootPid = Get-PlayerRootPid if ($rootPid) { Write-Log "Killing the hung player (pid $rootPid) and its children..." [void](Stop-PlayerTree -RootPid $rootPid) } $hasBeenHealthy = $false [void]$failureTimestamps.Add((Get-Date)) if (Test-Path -LiteralPath $StopFlagFile) { continue } [void](Invoke-CrashLoopBackoff -Failures $failureTimestamps) Write-Log ("Waiting ${RestartDelaySec}s before restarting...") Start-Sleep -Seconds $RestartDelaySec continue } # ---- Running but never healthy: be patient, then give up ----------- # Deliberately generous: a slow machine still inside its start-up window # must not be killed, and neither must the first-run setup screen. $stuckLimitSec = $StartupGraceSec * 2 if ($null -eq $launchTime) { $launchTime = Get-Date } $upSec = ((Get-Date) - $launchTime).TotalSeconds if ($upSec -lt $stuckLimitSec) { Write-Log ("Player is up but not yet healthy ({0}); still inside the ${stuckLimitSec}s start-up allowance." -f $ageText) 'WARN' Start-Sleep -Seconds $HealthCheckIntervalSec continue } Write-Log ("Player has not become healthy within ${stuckLimitSec}s ({0}) - restarting it." -f $ageText) 'ERROR' $rootPid = Get-PlayerRootPid if ($rootPid) { [void](Stop-PlayerTree -RootPid $rootPid) } [void]$failureTimestamps.Add((Get-Date)) [void](Invoke-CrashLoopBackoff -Failures $failureTimestamps) Write-Log ("Waiting ${RestartDelaySec}s before retrying...") Start-Sleep -Seconds $RestartDelaySec } Write-Log 'Watchdog exited.'