ske087/digiserver-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sanitize codebase, reorganize docs, and add missing deploy files

Browse Source 
Remove dead code identified in docs/SANITIZATION-REVIEW.md:
- app/blueprints/content_old.py and app/blueprints/playlist.py
- app/models/group.py, app/utils/nginx_config_reader.py
- orphaned templates (content_list, edit_content, upload_content,
  player_page) and the related group/Template references

Result: 6 blueprints, 82 routes, no dead modules or orphan templates.

Add files that deploy.sh and docker-entrypoint.sh already require but
which were never tracked:
- https_manager.py       (referenced by deploy.sh, migrate_network.sh,
                          docker-entrypoint.sh)
- Caddyfile.example      (seeded by deploy.sh; its absence aborts deploy)

Relocate generated Graphify artifacts from graphify-out/ to
docs/graphify-out/ (110 files, no content change) and archive the
superseded docs under docs/.

Ignore hygiene:
- ignore ad-hoc .env backups (.env.bak*) — they contain live secrets
- keep the pre-sanitization snapshots (docs/legacy code/,
  docs/old_code_documentation/) on disk but out of the repo

Fix .env.example: drop a duplicated config block, genericize the
hardcoded host IP, and document HOSTNAME_INTERNAL.
This commit is contained in:
scheianu
2026-09-11 12:18:34 +03:00
parent 1c5186463a
commit 46602f1933
226 changed files with 3999 additions and 15737 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/utils/player_build.py
+281 -57
View File
@@ -8,12 +8,25 @@ Admins use the "Build player files" admin page to:
The SSH deployment flow then ships this staged directory to player devices, so
the version admins build here is exactly what gets deployed.
Performance note
----------------
The player repository is large (~200 MB) and a full clone takes ~90 s. Because
the build runs inside an HTTP request, that would exceed gunicorn's worker
timeout and the worker would be killed mid-clone, leaving a broken checkout.
Two mitigations are used together:
* **Shallow clones** (``--depth 1``) — only the tip of the requested branch is
fetched, which is all a deployment needs. Drastically reduces transfer size.
* **Background execution** — the admin route starts the build in a daemon
thread and the page polls for progress, so no worker ever blocks on git.
"""
import os
import json
import shutil
import subprocess
import logging
import threading
from datetime import datetime
from typing import Any, Dict, Optional
@@ -24,90 +37,154 @@ logger = logging.getLogger(__name__)
# Metadata file name stored in the Flask instance folder.
BUILD_META_FILENAME = 'player_build.json'
# Only the tip of the branch is needed to deploy a player, so history is not
# fetched. Keeps the transfer small enough to avoid worker timeouts.
CLONE_DEPTH = '1'
def _run_git(args, cwd=None, timeout=300) -> subprocess.CompletedProcess:
return subprocess.run(
['git'] + args,
cwd=cwd,
capture_output=True,
text=True,
timeout=timeout,
)
# Never let git wait for a human. Without this, a private/renamed repository
# makes git block on a username prompt until the worker is killed.
GIT_ENV = {
'GIT_TERMINAL_PROMPT': '0', # never prompt for credentials
'GIT_ASKPASS': 'true', # answer any credential request immediately
'GIT_SSH_COMMAND': 'ssh -oBatchMode=yes -oStrictHostKeyChecking=accept-new',
}
def _git_env() -> Dict[str, str]:
"""Environment for git subprocesses: inherit the process env plus our flags."""
env = dict(os.environ)
env.update(GIT_ENV)
return env
def _run_git(args, cwd=None, timeout=120) -> subprocess.CompletedProcess:
"""Run a git command, never prompting for input.
Args:
args: git arguments (without the leading 'git').
cwd: working directory for the command.
timeout: hard cap in seconds. Defaults to 120 to stay within a
reasonable window even when running in the foreground.
Returns:
The completed process. ``returncode`` is 124 on timeout so callers can
distinguish a timeout from a normal failure.
"""
try:
return subprocess.run(
['git'] + args,
cwd=cwd,
capture_output=True,
text=True,
timeout=timeout,
env=_git_env(),
)
except subprocess.TimeoutExpired as e:
# Surface timeouts as a normal result so callers do not need try/except.
out = e.stdout.decode() if isinstance(e.stdout, bytes) else (e.stdout or '')
err = e.stderr.decode() if isinstance(e.stderr, bytes) else (e.stderr or '')
return subprocess.CompletedProcess(
args=['git'] + list(args), returncode=124,
stdout=out, stderr=(err + f'\ngit {" ".join(args)} timed out after {timeout}s').strip(),
)
def get_short_head(player_code_dir: str) -> str:
"""Return the short git commit of the staged code, or 'unknown'."""
try:
result = _run_git(['-C', player_code_dir, 'rev-parse', '--short', 'HEAD'], timeout=10)
if result.returncode == 0:
return result.stdout.strip()
except Exception:
pass
result = _run_git(['-C', player_code_dir, 'rev-parse', '--short', 'HEAD'], timeout=10)
if result.returncode == 0:
return result.stdout.strip()
return 'unknown'
def is_valid_checkout(path: str) -> bool:
"""True when *path* is a usable git checkout with a resolvable HEAD."""
if not os.path.isdir(os.path.join(path, '.git')):
return False
return get_short_head(path) != 'unknown'
def _clone(path: str, repo_url: str, branch: str) -> subprocess.CompletedProcess:
"""Shallow-clone a single branch into *path*."""
return _run_git([
'clone', '--depth', CLONE_DEPTH, '--single-branch',
'--branch', branch, repo_url, path,
], timeout=600)
def build_player_files(player_code_dir: str, repo_url: str, branch: str = 'main') -> Dict[str, Any]:
"""Clone or refresh the player source into ``player_code_dir``.
If the directory is already a git checkout of ``repo_url`` it is updated in
place (fetch + hard reset to the chosen branch). Otherwise it is cloned
fresh (an existing non-git directory is replaced).
Uses a **shallow single-branch clone/update** so only the tip of the wanted
branch is transferred. If the directory is a usable checkout it is updated
(fetch + hard reset to the branch). A directory that exists but is NOT a
usable checkout — e.g. left behind by an interrupted clone — is removed and
re-cloned, since updating it can never work.
Returns a dict: ``success`` (bool), ``message`` (str), ``version`` (str),
``branch`` (str).
Args:
player_code_dir: Destination directory for the staged player code.
repo_url: Git repository to pull from.
branch: Branch to stage.
Returns:
``{'success': bool, 'message': str, 'version': str|None, 'branch': str}``
"""
branch = (branch or 'main').strip()
repo_url = (repo_url or '').strip()
def fail(message: str) -> Dict[str, Any]:
return {'success': False, 'message': message,
'version': get_short_head(player_code_dir), 'branch': branch}
if not repo_url:
return {'success': False, 'message': 'Repository URL is required.', 'version': None, 'branch': branch}
return {'success': False, 'message': 'Repository URL is required.',
'version': None, 'branch': branch}
try:
git_dir = os.path.join(player_code_dir, '.git')
is_git_repo = os.path.isdir(git_dir)
usable = is_valid_checkout(player_code_dir)
if is_git_repo:
# Update existing checkout in place.
fetch = _run_git(['-C', player_code_dir, 'fetch', '--prune', 'origin'])
if usable:
# Update in place. Depth 1 keeps the update cheap; fetch by ref so
# it works on a shallow clone.
fetch = _run_git(
['-C', player_code_dir, 'fetch', '--depth', CLONE_DEPTH,
'--prune', 'origin', branch])
if fetch.returncode != 0:
return {
'success': False,
'message': f'git fetch failed: {fetch.stderr.strip() or fetch.stdout.strip()}',
'version': get_short_head(player_code_dir),
'branch': branch,
}
return fail(f'git fetch failed: {fetch.stderr.strip() or fetch.stdout.strip()}')
# Point origin at the requested URL in case it changed.
_run_git(['-C', player_code_dir, 'remote', 'set-url', 'origin', repo_url])
checkout = _run_git(['-C', player_code_dir, 'checkout', branch])
if checkout.returncode != 0:
return {
'success': False,
'message': f'git checkout {branch} failed: {checkout.stderr.strip()}',
'version': get_short_head(player_code_dir),
'branch': branch,
}
return fail(f'git checkout {branch} failed: {checkout.stderr.strip()}')
reset = _run_git(['-C', player_code_dir, 'reset', '--hard', f'origin/{branch}'])
if reset.returncode != 0:
return {
'success': False,
'message': f'git reset failed: {reset.stderr.strip()}',
'version': get_short_head(player_code_dir),
'branch': branch,
}
return fail(f'git reset failed: {reset.stderr.strip()}')
action = 'Updated'
else:
# Fresh clone. Replace any existing (non-git) directory.
# Fresh clone. A previous attempt may have left a partial directory
# (e.g. killed mid-clone) — it must go, or the clone will fail with
# "destination path already exists and is not an empty directory".
parent = os.path.dirname(player_code_dir.rstrip('/'))
os.makedirs(parent, exist_ok=True)
if parent:
os.makedirs(parent, exist_ok=True)
if os.path.exists(player_code_dir):
shutil.rmtree(player_code_dir)
clone = _run_git(['clone', '--branch', branch, repo_url, player_code_dir])
logger.info('Removing unusable directory before clone: %s', player_code_dir)
shutil.rmtree(player_code_dir, ignore_errors=True)
clone = _clone(player_code_dir, repo_url, branch)
if clone.returncode != 0:
return {
'success': False,
'message': f'git clone failed: {clone.stderr.strip() or clone.stdout.strip()}',
'version': None,
'branch': branch,
}
# Do not leave a half-written directory behind.
shutil.rmtree(player_code_dir, ignore_errors=True)
detail = clone.stderr.strip() or clone.stdout.strip()
if clone.returncode == 124 or 'timed out' in detail:
return fail(f'git clone timed out. The repository may be very '
f'large or unreachable: {detail}')
return fail(f'git clone failed: {detail}')
action = 'Cloned'
version = get_short_head(player_code_dir)
@@ -118,11 +195,15 @@ def build_player_files(player_code_dir: str, repo_url: str, branch: str = 'main'
'version': version,
'branch': branch,
}
except subprocess.TimeoutExpired:
return {'success': False, 'message': 'Git operation timed out.', 'version': None, 'branch': branch}
except Exception as e:
except Exception as e: # noqa: BLE001 - surface any failure
logger.exception('build_player_files failed')
return {'success': False, 'message': f'Build failed: {str(e)}', 'version': None, 'branch': branch}
# Never leave a broken checkout behind for the next attempt.
try:
if not is_valid_checkout(player_code_dir):
shutil.rmtree(player_code_dir, ignore_errors=True)
except Exception:
pass
return fail(f'Build failed: {str(e)}')
def write_base_config(
@@ -221,3 +302,146 @@ def make_build_record(repo_url, branch, server_ip, port, use_https, verify_ssl,
'built_at': datetime.utcnow().isoformat(timespec='seconds') + 'Z',
'built_by': built_by,
}
# ---------------------------------------------------------------------------
# Background builds
#
# A full clone/refresh takes far longer than gunicorn's worker timeout, so the
# build must not run inside the request. The admin route starts it here and the
# page polls `build_state()` for progress.
# ---------------------------------------------------------------------------
# Serialises writes to _build_state between the request thread and the worker.
_build_lock = threading.Lock()
# Coarse progress for the admin UI. 'state' is one of:
# idle | running | success | error
_build_state: Dict[str, Any] = {'state': 'idle'}
def get_build_state() -> Dict[str, Any]:
"""Return a snapshot of the current/last build for the admin UI."""
with _build_lock:
return dict(_build_state)
def is_build_running() -> bool:
"""True while a build is in progress."""
with _build_lock:
return _build_state.get('state') == 'running'
def _set_build_state(**fields: Any) -> None:
with _build_lock:
_build_state.update(fields)
def _run_build_job(app, player_code_dir: str, repo_url: str, branch: str,
config_payload: Optional[Dict[str, Any]],
meta_path: str, built_by: str) -> None:
"""Worker body: build files, optionally write config, then persist settings.
Runs in a daemon thread with its own Flask app context so it is independent
of the request/response cycle that triggered it.
"""
started = datetime.utcnow()
try:
_set_build_state(state='running', step='Fetching player source…',
started_at=started.isoformat(timespec='seconds') + 'Z',
message='', version=None)
result = build_player_files(player_code_dir, repo_url, branch)
version = result.get('version')
if not result['success']:
_set_build_state(state='error', step='', message=result['message'],
version=version,
finished_at=datetime.utcnow().isoformat(timespec='seconds') + 'Z')
logger.error('Background player build failed: %s', result['message'])
return
# Optional step 2: write the base config.
if config_payload:
_set_build_state(step='Writing player config…')
cfg = write_base_config(player_code_dir=player_code_dir, **config_payload)
if not cfg['success']:
_set_build_state(state='error', step='', message=cfg['message'],
version=version,
finished_at=datetime.utcnow().isoformat(timespec='seconds') + 'Z')
logger.error('Background player config write failed: %s', cfg['message'])
return
result = {**result, 'message': f"{result['message']} {cfg['message']}"}
if version is None:
version = get_short_head(player_code_dir)
save_build_settings(
meta_path,
make_build_record(
repo_url=repo_url, branch=branch,
server_ip=(config_payload or {}).get('server_ip', ''),
port=(config_payload or {}).get('port', ''),
use_https=(config_payload or {}).get('use_https', False),
verify_ssl=(config_payload or {}).get('verify_ssl', False),
orientation=(config_payload or {}).get('orientation', 'Landscape'),
max_resolution=(config_payload or {}).get('max_resolution', '1920x1080'),
version=version, built_by=built_by,
),
)
_set_build_state(state='success', step='', message=result['message'],
version=version,
finished_at=datetime.utcnow().isoformat(timespec='seconds') + 'Z')
logger.info('Background player build complete (version %s)', version)
except Exception as e: # noqa: BLE001 - never kill the thread silently
logger.exception('Background player build crashed')
_set_build_state(state='error', step='', message=f'Build failed: {e}',
finished_at=datetime.utcnow().isoformat(timespec='seconds') + 'Z')
def start_background_build(player_code_dir: str, repo_url: str, branch: str,
config_payload: Optional[Dict[str, Any]],
meta_path: str, built_by: str) -> bool:
"""Start a player build in a daemon thread.
Args:
player_code_dir: Where to stage the player source.
repo_url: Git repository URL.
branch: Branch to stage.
config_payload: Keyword args for :func:`write_base_config`, or None to
skip writing the config.
meta_path: Where to persist the build record.
built_by: Username shown in the UI/logs.
Returns:
False if a build is already running (callers should tell the user),
True if a new build was started.
Raises:
RuntimeError: if called with no Flask application context — the worker
thread needs a real app object to push its own context.
"""
from flask import current_app
if is_build_running():
return False
# Capture the real app object now. `current_app` resolves inside either a
# request or a plain application context; the worker thread pushes its own
# context later, since the caller's context is gone by then.
app = current_app._get_current_object()
_set_build_state(state='running', step='Starting…', message='', version=None,
started_at=datetime.utcnow().isoformat(timespec='seconds') + 'Z',
finished_at=None, built_by=built_by,
repo_url=repo_url, branch=branch)
thread = threading.Thread(
target=_run_build_job,
args=(app, player_code_dir, repo_url, branch, config_payload, meta_path, built_by),
name='player-build',
daemon=True,
)
thread.start()
return True