ske087
46602f1933
Remove dead code identified in docs/SANITIZATION-REVIEW.md:
- app/blueprints/content_old.py and app/blueprints/playlist.py
- app/models/group.py, app/utils/nginx_config_reader.py
- orphaned templates (content_list, edit_content, upload_content,
player_page) and the related group/Template references
Result: 6 blueprints, 82 routes, no dead modules or orphan templates.
Add files that deploy.sh and docker-entrypoint.sh already require but
which were never tracked:
- https_manager.py (referenced by deploy.sh, migrate_network.sh,
docker-entrypoint.sh)
- Caddyfile.example (seeded by deploy.sh; its absence aborts deploy)
Relocate generated Graphify artifacts from graphify-out/ to
docs/graphify-out/ (110 files, no content change) and archive the
superseded docs under docs/.
Ignore hygiene:
- ignore ad-hoc .env backups (.env.bak*) — they contain live secrets
- keep the pre-sanitization snapshots (docs/legacy code/,
docs/old_code_documentation/) on disk but out of the repo
Fix .env.example: drop a duplicated config block, genericize the
hardcoded host IP, and document HOSTNAME_INTERNAL.
448 lines
18 KiB
Python
448 lines
18 KiB
Python
"""Utilities for building/staging the player files on the server.
|
|
|
|
Admins use the "Build player files" admin page to:
|
|
* clone/refresh the player source code from a git repository into a local
|
|
staged directory (``PLAYER_CODE_DIR``), and
|
|
* write a base ``config/app_config.json`` so the staged code already knows how
|
|
to reach this server.
|
|
|
|
The SSH deployment flow then ships this staged directory to player devices, so
|
|
the version admins build here is exactly what gets deployed.
|
|
|
|
Performance note
|
|
----------------
|
|
The player repository is large (~200 MB) and a full clone takes ~90 s. Because
|
|
the build runs inside an HTTP request, that would exceed gunicorn's worker
|
|
timeout and the worker would be killed mid-clone, leaving a broken checkout.
|
|
Two mitigations are used together:
|
|
|
|
* **Shallow clones** (``--depth 1``) — only the tip of the requested branch is
|
|
fetched, which is all a deployment needs. Drastically reduces transfer size.
|
|
* **Background execution** — the admin route starts the build in a daemon
|
|
thread and the page polls for progress, so no worker ever blocks on git.
|
|
"""
|
|
import os
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
import logging
|
|
import threading
|
|
from datetime import datetime
|
|
from typing import Any, Dict, Optional
|
|
|
|
from app.utils.ssh_deploy import generate_app_config
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Metadata file name stored in the Flask instance folder.
|
|
BUILD_META_FILENAME = 'player_build.json'
|
|
|
|
# Only the tip of the branch is needed to deploy a player, so history is not
|
|
# fetched. Keeps the transfer small enough to avoid worker timeouts.
|
|
CLONE_DEPTH = '1'
|
|
|
|
# Never let git wait for a human. Without this, a private/renamed repository
|
|
# makes git block on a username prompt until the worker is killed.
|
|
GIT_ENV = {
|
|
'GIT_TERMINAL_PROMPT': '0', # never prompt for credentials
|
|
'GIT_ASKPASS': 'true', # answer any credential request immediately
|
|
'GIT_SSH_COMMAND': 'ssh -oBatchMode=yes -oStrictHostKeyChecking=accept-new',
|
|
}
|
|
|
|
|
|
def _git_env() -> Dict[str, str]:
|
|
"""Environment for git subprocesses: inherit the process env plus our flags."""
|
|
env = dict(os.environ)
|
|
env.update(GIT_ENV)
|
|
return env
|
|
|
|
|
|
def _run_git(args, cwd=None, timeout=120) -> subprocess.CompletedProcess:
|
|
"""Run a git command, never prompting for input.
|
|
|
|
Args:
|
|
args: git arguments (without the leading 'git').
|
|
cwd: working directory for the command.
|
|
timeout: hard cap in seconds. Defaults to 120 to stay within a
|
|
reasonable window even when running in the foreground.
|
|
|
|
Returns:
|
|
The completed process. ``returncode`` is 124 on timeout so callers can
|
|
distinguish a timeout from a normal failure.
|
|
"""
|
|
try:
|
|
return subprocess.run(
|
|
['git'] + args,
|
|
cwd=cwd,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=timeout,
|
|
env=_git_env(),
|
|
)
|
|
except subprocess.TimeoutExpired as e:
|
|
# Surface timeouts as a normal result so callers do not need try/except.
|
|
out = e.stdout.decode() if isinstance(e.stdout, bytes) else (e.stdout or '')
|
|
err = e.stderr.decode() if isinstance(e.stderr, bytes) else (e.stderr or '')
|
|
return subprocess.CompletedProcess(
|
|
args=['git'] + list(args), returncode=124,
|
|
stdout=out, stderr=(err + f'\ngit {" ".join(args)} timed out after {timeout}s').strip(),
|
|
)
|
|
|
|
|
|
def get_short_head(player_code_dir: str) -> str:
|
|
"""Return the short git commit of the staged code, or 'unknown'."""
|
|
result = _run_git(['-C', player_code_dir, 'rev-parse', '--short', 'HEAD'], timeout=10)
|
|
if result.returncode == 0:
|
|
return result.stdout.strip()
|
|
return 'unknown'
|
|
|
|
|
|
def is_valid_checkout(path: str) -> bool:
|
|
"""True when *path* is a usable git checkout with a resolvable HEAD."""
|
|
if not os.path.isdir(os.path.join(path, '.git')):
|
|
return False
|
|
return get_short_head(path) != 'unknown'
|
|
|
|
|
|
def _clone(path: str, repo_url: str, branch: str) -> subprocess.CompletedProcess:
|
|
"""Shallow-clone a single branch into *path*."""
|
|
return _run_git([
|
|
'clone', '--depth', CLONE_DEPTH, '--single-branch',
|
|
'--branch', branch, repo_url, path,
|
|
], timeout=600)
|
|
|
|
|
|
def build_player_files(player_code_dir: str, repo_url: str, branch: str = 'main') -> Dict[str, Any]:
|
|
"""Clone or refresh the player source into ``player_code_dir``.
|
|
|
|
Uses a **shallow single-branch clone/update** so only the tip of the wanted
|
|
branch is transferred. If the directory is a usable checkout it is updated
|
|
(fetch + hard reset to the branch). A directory that exists but is NOT a
|
|
usable checkout — e.g. left behind by an interrupted clone — is removed and
|
|
re-cloned, since updating it can never work.
|
|
|
|
Args:
|
|
player_code_dir: Destination directory for the staged player code.
|
|
repo_url: Git repository to pull from.
|
|
branch: Branch to stage.
|
|
|
|
Returns:
|
|
``{'success': bool, 'message': str, 'version': str|None, 'branch': str}``
|
|
"""
|
|
branch = (branch or 'main').strip()
|
|
repo_url = (repo_url or '').strip()
|
|
|
|
def fail(message: str) -> Dict[str, Any]:
|
|
return {'success': False, 'message': message,
|
|
'version': get_short_head(player_code_dir), 'branch': branch}
|
|
|
|
if not repo_url:
|
|
return {'success': False, 'message': 'Repository URL is required.',
|
|
'version': None, 'branch': branch}
|
|
|
|
try:
|
|
usable = is_valid_checkout(player_code_dir)
|
|
|
|
if usable:
|
|
# Update in place. Depth 1 keeps the update cheap; fetch by ref so
|
|
# it works on a shallow clone.
|
|
fetch = _run_git(
|
|
['-C', player_code_dir, 'fetch', '--depth', CLONE_DEPTH,
|
|
'--prune', 'origin', branch])
|
|
if fetch.returncode != 0:
|
|
return fail(f'git fetch failed: {fetch.stderr.strip() or fetch.stdout.strip()}')
|
|
|
|
# Point origin at the requested URL in case it changed.
|
|
_run_git(['-C', player_code_dir, 'remote', 'set-url', 'origin', repo_url])
|
|
|
|
checkout = _run_git(['-C', player_code_dir, 'checkout', branch])
|
|
if checkout.returncode != 0:
|
|
return fail(f'git checkout {branch} failed: {checkout.stderr.strip()}')
|
|
|
|
reset = _run_git(['-C', player_code_dir, 'reset', '--hard', f'origin/{branch}'])
|
|
if reset.returncode != 0:
|
|
return fail(f'git reset failed: {reset.stderr.strip()}')
|
|
|
|
action = 'Updated'
|
|
else:
|
|
# Fresh clone. A previous attempt may have left a partial directory
|
|
# (e.g. killed mid-clone) — it must go, or the clone will fail with
|
|
# "destination path already exists and is not an empty directory".
|
|
parent = os.path.dirname(player_code_dir.rstrip('/'))
|
|
if parent:
|
|
os.makedirs(parent, exist_ok=True)
|
|
if os.path.exists(player_code_dir):
|
|
logger.info('Removing unusable directory before clone: %s', player_code_dir)
|
|
shutil.rmtree(player_code_dir, ignore_errors=True)
|
|
|
|
clone = _clone(player_code_dir, repo_url, branch)
|
|
if clone.returncode != 0:
|
|
# Do not leave a half-written directory behind.
|
|
shutil.rmtree(player_code_dir, ignore_errors=True)
|
|
detail = clone.stderr.strip() or clone.stdout.strip()
|
|
if clone.returncode == 124 or 'timed out' in detail:
|
|
return fail(f'git clone timed out. The repository may be very '
|
|
f'large or unreachable: {detail}')
|
|
return fail(f'git clone failed: {detail}')
|
|
|
|
action = 'Cloned'
|
|
|
|
version = get_short_head(player_code_dir)
|
|
logger.info('%s player code from %s (%s) -> %s', action, repo_url, branch, version)
|
|
return {
|
|
'success': True,
|
|
'message': f'{action} player code from {branch} (version {version}).',
|
|
'version': version,
|
|
'branch': branch,
|
|
}
|
|
except Exception as e: # noqa: BLE001 - surface any failure
|
|
logger.exception('build_player_files failed')
|
|
# Never leave a broken checkout behind for the next attempt.
|
|
try:
|
|
if not is_valid_checkout(player_code_dir):
|
|
shutil.rmtree(player_code_dir, ignore_errors=True)
|
|
except Exception:
|
|
pass
|
|
return fail(f'Build failed: {str(e)}')
|
|
|
|
|
|
def write_base_config(
|
|
player_code_dir: str,
|
|
server_ip: str,
|
|
port: str,
|
|
use_https: bool = True,
|
|
verify_ssl: bool = False,
|
|
orientation: str = 'Landscape',
|
|
max_resolution: str = '1920x1080',
|
|
) -> Dict[str, Any]:
|
|
"""Write a base ``config/app_config.json`` into the staged player code.
|
|
|
|
``screen_name`` and ``quickconnect_key`` are left blank on purpose: they are
|
|
per-player and get filled in by the SSH deploy step for each device.
|
|
"""
|
|
try:
|
|
config_dir = os.path.join(player_code_dir, 'config')
|
|
os.makedirs(config_dir, exist_ok=True)
|
|
content = generate_app_config(
|
|
server_ip=server_ip,
|
|
port=str(port),
|
|
screen_name='',
|
|
quickconnect_code='',
|
|
orientation=orientation,
|
|
use_https=use_https,
|
|
verify_ssl=verify_ssl,
|
|
max_resolution=max_resolution,
|
|
)
|
|
config_path = os.path.join(config_dir, 'app_config.json')
|
|
with open(config_path, 'w', encoding='utf-8') as f:
|
|
f.write(content)
|
|
logger.info('Wrote base player config -> %s', config_path)
|
|
return {'success': True, 'message': 'Base player config written.', 'path': config_path}
|
|
except Exception as e:
|
|
logger.exception('write_base_config failed')
|
|
return {'success': False, 'message': f'Failed to write config: {str(e)}'}
|
|
|
|
|
|
def load_build_settings(meta_path: str) -> Optional[Dict[str, Any]]:
|
|
"""Load saved build settings from ``meta_path`` (or None if absent/invalid)."""
|
|
try:
|
|
if os.path.isfile(meta_path):
|
|
with open(meta_path, 'r', encoding='utf-8') as f:
|
|
return json.load(f)
|
|
except Exception as e:
|
|
logger.warning('Could not read build settings: %s', e)
|
|
return None
|
|
|
|
|
|
def save_build_settings(meta_path: str, data: Dict[str, Any]) -> bool:
|
|
"""Persist build settings to ``meta_path``."""
|
|
try:
|
|
os.makedirs(os.path.dirname(meta_path), exist_ok=True)
|
|
with open(meta_path, 'w', encoding='utf-8') as f:
|
|
json.dump(data, f, indent=2)
|
|
return True
|
|
except Exception as e:
|
|
logger.warning('Could not save build settings: %s', e)
|
|
return False
|
|
|
|
|
|
def get_player_server_settings(meta_path: str) -> Optional[Dict[str, Any]]:
|
|
"""Return the saved server address settings for deployment, if available.
|
|
|
|
Returns a dict with ``server_ip``, ``port`` (str), ``use_https`` (bool) and
|
|
``verify_ssl`` (bool), or None when no usable build settings are saved.
|
|
"""
|
|
settings = load_build_settings(meta_path)
|
|
if not settings:
|
|
return None
|
|
server_ip = (settings.get('server_ip') or '').strip()
|
|
if not server_ip:
|
|
return None
|
|
return {
|
|
'server_ip': server_ip,
|
|
'port': str(settings.get('port') or ('443' if settings.get('use_https', True) else '80')),
|
|
'use_https': bool(settings.get('use_https', True)),
|
|
'verify_ssl': bool(settings.get('verify_ssl', False)),
|
|
}
|
|
|
|
|
|
def make_build_record(repo_url, branch, server_ip, port, use_https, verify_ssl,
|
|
orientation, max_resolution, version, built_by) -> Dict[str, Any]:
|
|
"""Assemble the metadata record to persist after a build."""
|
|
return {
|
|
'repo_url': repo_url,
|
|
'branch': branch,
|
|
'server_ip': server_ip,
|
|
'port': str(port),
|
|
'use_https': bool(use_https),
|
|
'verify_ssl': bool(verify_ssl),
|
|
'orientation': orientation,
|
|
'max_resolution': max_resolution,
|
|
'built_version': version,
|
|
'built_at': datetime.utcnow().isoformat(timespec='seconds') + 'Z',
|
|
'built_by': built_by,
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Background builds
|
|
#
|
|
# A full clone/refresh takes far longer than gunicorn's worker timeout, so the
|
|
# build must not run inside the request. The admin route starts it here and the
|
|
# page polls `build_state()` for progress.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Serialises writes to _build_state between the request thread and the worker.
|
|
_build_lock = threading.Lock()
|
|
|
|
# Coarse progress for the admin UI. 'state' is one of:
|
|
# idle | running | success | error
|
|
_build_state: Dict[str, Any] = {'state': 'idle'}
|
|
|
|
|
|
def get_build_state() -> Dict[str, Any]:
|
|
"""Return a snapshot of the current/last build for the admin UI."""
|
|
with _build_lock:
|
|
return dict(_build_state)
|
|
|
|
|
|
def is_build_running() -> bool:
|
|
"""True while a build is in progress."""
|
|
with _build_lock:
|
|
return _build_state.get('state') == 'running'
|
|
|
|
|
|
def _set_build_state(**fields: Any) -> None:
|
|
with _build_lock:
|
|
_build_state.update(fields)
|
|
|
|
|
|
def _run_build_job(app, player_code_dir: str, repo_url: str, branch: str,
|
|
config_payload: Optional[Dict[str, Any]],
|
|
meta_path: str, built_by: str) -> None:
|
|
"""Worker body: build files, optionally write config, then persist settings.
|
|
|
|
Runs in a daemon thread with its own Flask app context so it is independent
|
|
of the request/response cycle that triggered it.
|
|
"""
|
|
started = datetime.utcnow()
|
|
try:
|
|
_set_build_state(state='running', step='Fetching player source…',
|
|
started_at=started.isoformat(timespec='seconds') + 'Z',
|
|
message='', version=None)
|
|
|
|
result = build_player_files(player_code_dir, repo_url, branch)
|
|
version = result.get('version')
|
|
|
|
if not result['success']:
|
|
_set_build_state(state='error', step='', message=result['message'],
|
|
version=version,
|
|
finished_at=datetime.utcnow().isoformat(timespec='seconds') + 'Z')
|
|
logger.error('Background player build failed: %s', result['message'])
|
|
return
|
|
|
|
# Optional step 2: write the base config.
|
|
if config_payload:
|
|
_set_build_state(step='Writing player config…')
|
|
cfg = write_base_config(player_code_dir=player_code_dir, **config_payload)
|
|
if not cfg['success']:
|
|
_set_build_state(state='error', step='', message=cfg['message'],
|
|
version=version,
|
|
finished_at=datetime.utcnow().isoformat(timespec='seconds') + 'Z')
|
|
logger.error('Background player config write failed: %s', cfg['message'])
|
|
return
|
|
result = {**result, 'message': f"{result['message']} {cfg['message']}"}
|
|
|
|
if version is None:
|
|
version = get_short_head(player_code_dir)
|
|
|
|
save_build_settings(
|
|
meta_path,
|
|
make_build_record(
|
|
repo_url=repo_url, branch=branch,
|
|
server_ip=(config_payload or {}).get('server_ip', ''),
|
|
port=(config_payload or {}).get('port', ''),
|
|
use_https=(config_payload or {}).get('use_https', False),
|
|
verify_ssl=(config_payload or {}).get('verify_ssl', False),
|
|
orientation=(config_payload or {}).get('orientation', 'Landscape'),
|
|
max_resolution=(config_payload or {}).get('max_resolution', '1920x1080'),
|
|
version=version, built_by=built_by,
|
|
),
|
|
)
|
|
|
|
_set_build_state(state='success', step='', message=result['message'],
|
|
version=version,
|
|
finished_at=datetime.utcnow().isoformat(timespec='seconds') + 'Z')
|
|
logger.info('Background player build complete (version %s)', version)
|
|
except Exception as e: # noqa: BLE001 - never kill the thread silently
|
|
logger.exception('Background player build crashed')
|
|
_set_build_state(state='error', step='', message=f'Build failed: {e}',
|
|
finished_at=datetime.utcnow().isoformat(timespec='seconds') + 'Z')
|
|
|
|
|
|
def start_background_build(player_code_dir: str, repo_url: str, branch: str,
|
|
config_payload: Optional[Dict[str, Any]],
|
|
meta_path: str, built_by: str) -> bool:
|
|
"""Start a player build in a daemon thread.
|
|
|
|
Args:
|
|
player_code_dir: Where to stage the player source.
|
|
repo_url: Git repository URL.
|
|
branch: Branch to stage.
|
|
config_payload: Keyword args for :func:`write_base_config`, or None to
|
|
skip writing the config.
|
|
meta_path: Where to persist the build record.
|
|
built_by: Username shown in the UI/logs.
|
|
|
|
Returns:
|
|
False if a build is already running (callers should tell the user),
|
|
True if a new build was started.
|
|
|
|
Raises:
|
|
RuntimeError: if called with no Flask application context — the worker
|
|
thread needs a real app object to push its own context.
|
|
"""
|
|
from flask import current_app
|
|
|
|
if is_build_running():
|
|
return False
|
|
|
|
# Capture the real app object now. `current_app` resolves inside either a
|
|
# request or a plain application context; the worker thread pushes its own
|
|
# context later, since the caller's context is gone by then.
|
|
app = current_app._get_current_object()
|
|
|
|
_set_build_state(state='running', step='Starting…', message='', version=None,
|
|
started_at=datetime.utcnow().isoformat(timespec='seconds') + 'Z',
|
|
finished_at=None, built_by=built_by,
|
|
repo_url=repo_url, branch=branch)
|
|
|
|
thread = threading.Thread(
|
|
target=_run_build_job,
|
|
args=(app, player_code_dir, repo_url, branch, config_payload, meta_path, built_by),
|
|
name='player-build',
|
|
daemon=True,
|
|
)
|
|
thread.start()
|
|
return True
|