Sanitize codebase, reorganize docs, and add missing deploy files
Remove dead code identified in docs/SANITIZATION-REVIEW.md:
- app/blueprints/content_old.py and app/blueprints/playlist.py
- app/models/group.py, app/utils/nginx_config_reader.py
- orphaned templates (content_list, edit_content, upload_content,
player_page) and the related group/Template references
Result: 6 blueprints, 82 routes, no dead modules or orphan templates.
Add files that deploy.sh and docker-entrypoint.sh already require but
which were never tracked:
- https_manager.py (referenced by deploy.sh, migrate_network.sh,
docker-entrypoint.sh)
- Caddyfile.example (seeded by deploy.sh; its absence aborts deploy)
Relocate generated Graphify artifacts from graphify-out/ to
docs/graphify-out/ (110 files, no content change) and archive the
superseded docs under docs/.
Ignore hygiene:
- ignore ad-hoc .env backups (.env.bak*) — they contain live secrets
- keep the pre-sanitization snapshots (docs/legacy code/,
docs/old_code_documentation/) on disk but out of the repo
Fix .env.example: drop a duplicated config block, genericize the
hardcoded host IP, and document HOSTNAME_INTERNAL.
This commit is contained in:
+2
-14
@@ -10,14 +10,7 @@ from app.utils.uploads import (
|
||||
get_file_size,
|
||||
delete_file
|
||||
)
|
||||
from app.utils.group_player_management import (
|
||||
get_player_status_info,
|
||||
get_group_statistics,
|
||||
assign_player_to_group,
|
||||
bulk_assign_players_to_group,
|
||||
get_online_players_count,
|
||||
get_players_by_status
|
||||
)
|
||||
from app.utils.group_player_management import get_player_status_info
|
||||
from app.utils.pptx_converter import pptx_to_pdf_libreoffice, validate_pptx_file
|
||||
|
||||
__all__ = [
|
||||
@@ -36,13 +29,8 @@ __all__ = [
|
||||
'clear_upload_progress',
|
||||
'get_file_size',
|
||||
'delete_file',
|
||||
# Group/Player Management
|
||||
# Player Management
|
||||
'get_player_status_info',
|
||||
'get_group_statistics',
|
||||
'assign_player_to_group',
|
||||
'bulk_assign_players_to_group',
|
||||
'get_online_players_count',
|
||||
'get_players_by_status',
|
||||
# PPTX Converter
|
||||
'pptx_to_pdf_libreoffice',
|
||||
'validate_pptx_file',
|
||||
|
||||
+93
-170
@@ -37,43 +37,110 @@ class CaddyConfigGenerator:
|
||||
"""Generate Caddyfile configuration based on HTTPSConfig."""
|
||||
|
||||
@staticmethod
|
||||
def generate_caddyfile(config: Optional['HTTPSConfig'] = None) -> str:
|
||||
def generate_caddyfile(config: Optional['HTTPSConfig'] = None,
|
||||
http_fallback: bool = True,
|
||||
http_port: int = 80,
|
||||
https_port: int = 443) -> str:
|
||||
"""Generate a complete Caddyfile.
|
||||
|
||||
Behaviour:
|
||||
- HTTPS disabled / no domain → HTTP-only on port 80 (initial deploy mode).
|
||||
- HTTPS enabled + real domain → Caddy auto-provisions a Let's Encrypt cert
|
||||
for that domain; HTTP redirects to HTTPS automatically.
|
||||
- HTTPS enabled + IP only (no domain) → TLS with Caddy's internal CA
|
||||
(self-signed, trusted within the Docker network).
|
||||
Design goals
|
||||
------------
|
||||
* **One HTTP endpoint** (port 80) that always answers, whatever the Host
|
||||
header is — so ``http://<ip>`` and ``http://<hostname>`` both work.
|
||||
* **HTTPS on port 443** for the same names when it is enabled.
|
||||
* If HTTPS is disabled or never configured, port 80 simply serves the
|
||||
app — there is no separate "HTTP mode" to configure.
|
||||
|
||||
Behaviour by configuration
|
||||
--------------------------
|
||||
* HTTPS off, or no address configured → plain HTTP on ``:http_port``.
|
||||
* HTTPS on → the app is served on port 80 for every configured name and
|
||||
on port 443 over TLS. Whether port 80 *serves* or *redirects* to
|
||||
HTTPS is controlled by ``http_fallback``.
|
||||
|
||||
Which certificate each name gets
|
||||
--------------------------------
|
||||
* ``domain`` (when set) → Caddy obtains a certificate automatically
|
||||
(Let's Encrypt/ACME). Only valid for a **publicly resolvable** name.
|
||||
* ``ip_address`` / ``hostname`` → ``tls internal`` (Caddy's local CA).
|
||||
This needs no public DNS and no ACME, which is the right choice for an
|
||||
intranet name such as ``digiserver.sibiusb.harting.intra``.
|
||||
|
||||
Args:
|
||||
config: HTTPSConfig instance, or None to load from the database.
|
||||
http_fallback: When True, port 80 keeps *serving* the app alongside
|
||||
HTTPS. This is the resilient default: clients that cannot trust
|
||||
the internal CA (e.g. a Kivy player with ``verify_ssl: true``)
|
||||
are still able to connect. When False, port 80 issues a 301
|
||||
redirect to HTTPS instead.
|
||||
http_port: Port Caddy listens on for plain HTTP (default 80).
|
||||
https_port: Port used to build redirect targets when
|
||||
``http_fallback`` is False (default 443).
|
||||
|
||||
Returns:
|
||||
The complete Caddyfile as a string.
|
||||
"""
|
||||
if config is None:
|
||||
config = HTTPSConfig.get_config()
|
||||
|
||||
email = (config.email or "admin@localhost") if config else "admin@localhost"
|
||||
https_enabled = config.https_enabled if config else False
|
||||
https_enabled = bool(config.https_enabled) if config else False
|
||||
domain = (config.domain or "").strip() if config else ""
|
||||
ip_address = (config.ip_address or "").strip() if config else ""
|
||||
hostname = (config.hostname or "").strip() if config else ""
|
||||
|
||||
global_block = f"""{{\n admin 0.0.0.0:2019\n email {email}\n}}\n\n"""
|
||||
# Every name the server should answer to, in priority order, without
|
||||
# duplicates. The IP comes first because it always resolves.
|
||||
names: list[str] = []
|
||||
for candidate in (ip_address, hostname, domain):
|
||||
if candidate and candidate not in names:
|
||||
names.append(candidate)
|
||||
|
||||
if https_enabled and domain:
|
||||
# Caddy handles Let's Encrypt + HTTP→HTTPS redirect automatically
|
||||
# when a plain hostname (no scheme) is used.
|
||||
caddyfile = global_block
|
||||
caddyfile += f"{domain} {{\n{_PROXY_SNIPPET}}}\n"
|
||||
# Also accept requests on the raw IP (HTTP only, no cert needed)
|
||||
if ip_address:
|
||||
caddyfile += f"\nhttp://{ip_address} {{\n{_PROXY_SNIPPET}}}\n"
|
||||
elif https_enabled and ip_address:
|
||||
# No public domain — use Caddy's internal CA (self-signed)
|
||||
caddyfile = global_block
|
||||
caddyfile += f"https://{ip_address} {{\n tls internal\n{_PROXY_SNIPPET}}}\n"
|
||||
caddyfile += f"\nhttp://{ip_address} {{\n redir https://{ip_address}{{uri}} 301\n}}\n"
|
||||
else:
|
||||
# HTTP-only fallback (first deploy, before HTTPS is configured)
|
||||
caddyfile = "{\n admin 0.0.0.0:2019\n}\n\n"
|
||||
caddyfile += f":80 {{\n{_PROXY_SNIPPET}}}\n"
|
||||
global_block = f"{{\n admin 0.0.0.0:2019\n email {email}\n"
|
||||
|
||||
# ── TLS with no SNI ────────────────────────────────────────────────
|
||||
# Browsers do NOT send SNI when the URL is an IP address (an IP is not
|
||||
# a valid SNI hostname). Without a fallback Caddy would identify such a
|
||||
# connection by the container's own internal IP, match no certificate
|
||||
# and abort the handshake with:
|
||||
# "no certificate available for '<container-ip>'"
|
||||
# `default_sni` makes a SNI-less ClientHello resolve to a name we do
|
||||
# serve, so https://<ip> works in the browser.
|
||||
if https_enabled and not domain and ip_address:
|
||||
global_block += f" default_sni {ip_address}\n"
|
||||
|
||||
global_block += "}\n\n"
|
||||
|
||||
# ── Plain HTTP only: HTTPS disabled, or no address to certify ───────
|
||||
if not (https_enabled and names):
|
||||
return global_block + f":{http_port} {{\n{_PROXY_SNIPPET}}}\n"
|
||||
|
||||
caddyfile = global_block
|
||||
|
||||
# ── Port 80: catch-all so ANY Host header is answered ──────────────
|
||||
# Without this, a request for an unexpected name (e.g. a bare IP when
|
||||
# only a hostname is configured) would hit no site block and fail.
|
||||
caddyfile += f":{http_port} {{\n{_PROXY_SNIPPET}}}\n\n"
|
||||
|
||||
# ── Port 80: explicit per-name blocks ──────────────────────────────
|
||||
for name in names:
|
||||
if http_fallback:
|
||||
caddyfile += f"http://{name} {{\n{_PROXY_SNIPPET}}}\n\n"
|
||||
else:
|
||||
# Redirect to the port the host actually publishes.
|
||||
https_url = (f"https://{name}" if https_port == 443
|
||||
else f"https://{name}:{https_port}")
|
||||
caddyfile += f"http://{name} {{\n redir {https_url}{{uri}} 301\n}}\n\n"
|
||||
|
||||
# ── Port 443: TLS listeners ────────────────────────────────────────
|
||||
for name in names:
|
||||
if domain and name == domain:
|
||||
# Public name → let Caddy obtain a real certificate.
|
||||
caddyfile += f"https://{name} {{\n{_PROXY_SNIPPET}}}\n\n"
|
||||
else:
|
||||
# IP or intranet name → Caddy's internal CA.
|
||||
caddyfile += (f"https://{name} {{\n tls internal\n"
|
||||
f"{_PROXY_SNIPPET}}}\n\n")
|
||||
|
||||
return caddyfile
|
||||
|
||||
@@ -121,148 +188,4 @@ class CaddyConfigGenerator:
|
||||
return response.status == 200
|
||||
except Exception as e:
|
||||
print(f"Caddy reload error: {str(e)}")
|
||||
return False
|
||||
|
||||
"""Generate complete Caddyfile content.
|
||||
|
||||
Args:
|
||||
config: HTTPSConfig instance or None
|
||||
|
||||
Returns:
|
||||
Complete Caddyfile content as string
|
||||
"""
|
||||
# Get config from database if not provided
|
||||
if config is None:
|
||||
config = HTTPSConfig.get_config()
|
||||
|
||||
# Base configuration
|
||||
email = "admin@localhost"
|
||||
if config and config.email:
|
||||
email = config.email
|
||||
|
||||
base_config = f"""{{
|
||||
# Global options
|
||||
email {email}
|
||||
# Admin API for configuration management (listen on all interfaces)
|
||||
admin 0.0.0.0:2019
|
||||
# Uncomment for testing to avoid rate limits
|
||||
# acme_ca https://acme-staging-v02.api.letsencrypt.org/directory
|
||||
}}
|
||||
|
||||
# Shared reverse proxy configuration
|
||||
(reverse_proxy_config) {{
|
||||
reverse_proxy digiserver-app:5000 {{
|
||||
header_up Host {{host}}
|
||||
header_up X-Real-IP {{remote_host}}
|
||||
header_up X-Forwarded-Proto {{scheme}}
|
||||
|
||||
# Timeouts for large uploads
|
||||
transport http {{
|
||||
read_timeout 300s
|
||||
write_timeout 300s
|
||||
}}
|
||||
}}
|
||||
|
||||
# File upload size limit (2GB)
|
||||
request_body {{
|
||||
max_size 2GB
|
||||
}}
|
||||
|
||||
# Security headers
|
||||
header {{
|
||||
X-Frame-Options "SAMEORIGIN"
|
||||
X-Content-Type-Options "nosniff"
|
||||
X-XSS-Protection "1; mode=block"
|
||||
}}
|
||||
|
||||
# Logging
|
||||
log {{
|
||||
output file /var/log/caddy/access.log
|
||||
}}
|
||||
}}
|
||||
|
||||
# Localhost (development/local access)
|
||||
http://localhost {{
|
||||
import reverse_proxy_config
|
||||
}}
|
||||
"""
|
||||
|
||||
# Add main domain/IP configuration if HTTPS is enabled
|
||||
if config and config.https_enabled and config.domain and config.ip_address:
|
||||
# Internal domain configuration
|
||||
domain_config = f"""
|
||||
# Internal domain (HTTP only - internal use)
|
||||
http://{config.domain} {{
|
||||
import reverse_proxy_config
|
||||
}}
|
||||
|
||||
# Handle IP address access
|
||||
http://{config.ip_address} {{
|
||||
import reverse_proxy_config
|
||||
}}
|
||||
"""
|
||||
base_config += domain_config
|
||||
else:
|
||||
# Default fallback configuration
|
||||
base_config += """
|
||||
# Internal domain (HTTP only - internal use)
|
||||
http://digiserver.sibiusb.harting.intra {
|
||||
import reverse_proxy_config
|
||||
}
|
||||
|
||||
# Handle IP address access
|
||||
http://10.76.152.164 {
|
||||
import reverse_proxy_config
|
||||
}
|
||||
"""
|
||||
|
||||
# Add catch-all for any other HTTP requests
|
||||
base_config += """
|
||||
# Catch-all for any other HTTP requests
|
||||
http://* {
|
||||
import reverse_proxy_config
|
||||
}
|
||||
"""
|
||||
|
||||
return base_config
|
||||
|
||||
@staticmethod
|
||||
def write_caddyfile(caddyfile_content: str, path: str = '/app/Caddyfile') -> bool:
|
||||
"""Write Caddyfile to disk.
|
||||
|
||||
Args:
|
||||
caddyfile_content: Content to write
|
||||
path: Path to Caddyfile
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
with open(path, 'w') as f:
|
||||
f.write(caddyfile_content)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"Error writing Caddyfile: {str(e)}")
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def reload_caddy() -> bool:
|
||||
"""Reload Caddy configuration without restart.
|
||||
|
||||
Note: Caddy monitoring is handled via file watching. After writing the Caddyfile,
|
||||
Caddy should automatically reload. If it doesn't, you may need to restart the
|
||||
Caddy container manually.
|
||||
|
||||
Returns:
|
||||
True if configuration was written successfully (Caddy will auto-reload)
|
||||
"""
|
||||
try:
|
||||
# Just verify that Caddy is reachable
|
||||
import urllib.request
|
||||
response = urllib.request.urlopen('http://caddy:2019/config/', timeout=2)
|
||||
return response.status == 200
|
||||
except Exception as e:
|
||||
# Caddy might not be reachable, but Caddyfile was already written
|
||||
# Caddy should reload automatically when it detects file changes
|
||||
print(f"Note: Caddy reload check returned: {str(e)}")
|
||||
return True # Return True anyway since Caddyfile was written
|
||||
|
||||
@@ -1,23 +1,26 @@
|
||||
"""Group and player management utilities."""
|
||||
from typing import Dict, List, Optional
|
||||
from datetime import datetime, timedelta
|
||||
"""Player status utilities.
|
||||
|
||||
from app.extensions import db
|
||||
from app.models import Player, Group, PlayerFeedback
|
||||
from app.utils.logger import log_action
|
||||
Note: the group-management helpers that used to live here were removed along
|
||||
with the deprecated Group subsystem (the ``group`` table had no rows and the
|
||||
``/api/groups`` endpoint had already been archived).
|
||||
"""
|
||||
from typing import Dict
|
||||
from datetime import datetime
|
||||
|
||||
from app.models import Player, PlayerFeedback
|
||||
|
||||
|
||||
def get_player_status_info(player_id: int) -> Dict:
|
||||
"""Get comprehensive status information for a player.
|
||||
|
||||
|
||||
Args:
|
||||
player_id: Player ID to query
|
||||
|
||||
|
||||
Returns:
|
||||
Dictionary with status information
|
||||
"""
|
||||
player = Player.query.get(player_id)
|
||||
|
||||
|
||||
if not player:
|
||||
return {
|
||||
'online': False,
|
||||
@@ -25,18 +28,18 @@ def get_player_status_info(player_id: int) -> Dict:
|
||||
'last_seen': None,
|
||||
'latest_feedback': None
|
||||
}
|
||||
|
||||
|
||||
# Check if player is online (seen in last 5 minutes)
|
||||
is_online = False
|
||||
if player.last_seen:
|
||||
delta = datetime.utcnow() - player.last_seen
|
||||
is_online = delta.total_seconds() < 300
|
||||
|
||||
|
||||
# Get latest feedback
|
||||
latest_feedback = PlayerFeedback.query.filter_by(player_id=player_id)\
|
||||
.order_by(PlayerFeedback.timestamp.desc())\
|
||||
.first()
|
||||
|
||||
|
||||
return {
|
||||
'online': is_online,
|
||||
'status': player.status,
|
||||
@@ -51,154 +54,18 @@ def get_player_status_info(player_id: int) -> Dict:
|
||||
}
|
||||
|
||||
|
||||
def get_group_statistics(group_id: int) -> Dict:
|
||||
"""Get statistics for a group.
|
||||
|
||||
Args:
|
||||
group_id: Group ID to query
|
||||
|
||||
Returns:
|
||||
Dictionary with group statistics
|
||||
"""
|
||||
group = Group.query.get(group_id)
|
||||
|
||||
if not group:
|
||||
return {
|
||||
'total_players': 0,
|
||||
'online_players': 0,
|
||||
'total_content': 0,
|
||||
'error_count': 0
|
||||
}
|
||||
|
||||
total_players = group.player_count
|
||||
total_content = group.content_count
|
||||
|
||||
# Count online players
|
||||
online_players = 0
|
||||
error_count = 0
|
||||
five_min_ago = datetime.utcnow() - timedelta(minutes=5)
|
||||
|
||||
for player in group.players:
|
||||
if player.last_seen and player.last_seen >= five_min_ago:
|
||||
online_players += 1
|
||||
if player.status == 'error':
|
||||
error_count += 1
|
||||
|
||||
return {
|
||||
'group_id': group_id,
|
||||
'group_name': group.name,
|
||||
'total_players': total_players,
|
||||
'online_players': online_players,
|
||||
'offline_players': total_players - online_players,
|
||||
'total_content': total_content,
|
||||
'error_count': error_count
|
||||
}
|
||||
|
||||
|
||||
def assign_player_to_group(player_id: int, group_id: Optional[int]) -> bool:
|
||||
"""Assign a player to a group or unassign if group_id is None.
|
||||
|
||||
Args:
|
||||
player_id: Player ID to assign
|
||||
group_id: Group ID to assign to, or None to unassign
|
||||
|
||||
Returns:
|
||||
True if successful, False otherwise
|
||||
"""
|
||||
try:
|
||||
player = Player.query.get(player_id)
|
||||
|
||||
if not player:
|
||||
log_action('error', f'Player {player_id} not found')
|
||||
return False
|
||||
|
||||
old_group_id = player.group_id
|
||||
player.group_id = group_id
|
||||
db.session.commit()
|
||||
|
||||
if group_id:
|
||||
group = Group.query.get(group_id)
|
||||
log_action('info', f'Player "{player.name}" assigned to group "{group.name}"')
|
||||
else:
|
||||
log_action('info', f'Player "{player.name}" unassigned from group')
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
log_action('error', f'Error assigning player to group: {str(e)}')
|
||||
return False
|
||||
|
||||
|
||||
def bulk_assign_players_to_group(player_ids: List[int], group_id: Optional[int]) -> int:
|
||||
"""Assign multiple players to a group.
|
||||
|
||||
Args:
|
||||
player_ids: List of player IDs to assign
|
||||
group_id: Group ID to assign to, or None to unassign
|
||||
|
||||
Returns:
|
||||
Number of players successfully assigned
|
||||
"""
|
||||
count = 0
|
||||
|
||||
try:
|
||||
for player_id in player_ids:
|
||||
player = Player.query.get(player_id)
|
||||
if player:
|
||||
player.group_id = group_id
|
||||
count += 1
|
||||
|
||||
db.session.commit()
|
||||
|
||||
if group_id:
|
||||
group = Group.query.get(group_id)
|
||||
log_action('info', f'Bulk assigned {count} players to group "{group.name}"')
|
||||
else:
|
||||
log_action('info', f'Bulk unassigned {count} players from groups')
|
||||
|
||||
return count
|
||||
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
log_action('error', f'Error bulk assigning players: {str(e)}')
|
||||
return 0
|
||||
|
||||
|
||||
def get_online_players_count() -> int:
|
||||
"""Get count of online players (seen in last 5 minutes).
|
||||
|
||||
Returns:
|
||||
Number of online players
|
||||
"""
|
||||
five_min_ago = datetime.utcnow() - timedelta(minutes=5)
|
||||
return Player.query.filter(Player.last_seen >= five_min_ago).count()
|
||||
|
||||
|
||||
def get_players_by_status(status: str) -> List[Player]:
|
||||
"""Get all players with a specific status.
|
||||
|
||||
Args:
|
||||
status: Status to filter by
|
||||
|
||||
Returns:
|
||||
List of Player instances
|
||||
"""
|
||||
return Player.query.filter_by(status=status).all()
|
||||
|
||||
|
||||
def _format_time_ago(dt: datetime) -> str:
|
||||
"""Format datetime as 'time ago' string.
|
||||
|
||||
|
||||
Args:
|
||||
dt: Datetime to format
|
||||
|
||||
|
||||
Returns:
|
||||
Formatted string like '5 minutes ago'
|
||||
"""
|
||||
delta = datetime.utcnow() - dt
|
||||
seconds = delta.total_seconds()
|
||||
|
||||
|
||||
if seconds < 60:
|
||||
return f'{int(seconds)} seconds ago'
|
||||
elif seconds < 3600:
|
||||
|
||||
@@ -1,120 +0,0 @@
|
||||
"""Nginx configuration reader utility."""
|
||||
import os
|
||||
import re
|
||||
from typing import Dict, List, Optional, Any
|
||||
|
||||
|
||||
class NginxConfigReader:
|
||||
"""Read and parse Nginx configuration files."""
|
||||
|
||||
def __init__(self, config_path: str = '/etc/nginx/nginx.conf'):
|
||||
"""Initialize Nginx config reader."""
|
||||
self.config_path = config_path
|
||||
self.config_content = None
|
||||
self.is_available = os.path.exists(config_path)
|
||||
|
||||
if self.is_available:
|
||||
try:
|
||||
with open(config_path, 'r') as f:
|
||||
self.config_content = f.read()
|
||||
except Exception as e:
|
||||
self.is_available = False
|
||||
self.error = str(e)
|
||||
|
||||
def get_status(self) -> Dict[str, Any]:
|
||||
"""Get Nginx configuration status."""
|
||||
if not self.is_available:
|
||||
return {
|
||||
'available': False,
|
||||
'error': 'Nginx configuration not found',
|
||||
'path': self.config_path
|
||||
}
|
||||
|
||||
return {
|
||||
'available': True,
|
||||
'path': self.config_path,
|
||||
'file_exists': os.path.exists(self.config_path),
|
||||
'ssl_enabled': self._check_ssl_enabled(),
|
||||
'http_ports': self._extract_http_ports(),
|
||||
'https_ports': self._extract_https_ports(),
|
||||
'upstream_servers': self._extract_upstream_servers(),
|
||||
'server_names': self._extract_server_names(),
|
||||
'ssl_protocols': self._extract_ssl_protocols(),
|
||||
'client_max_body_size': self._extract_client_max_body_size(),
|
||||
'gzip_enabled': self._check_gzip_enabled(),
|
||||
}
|
||||
|
||||
def _check_ssl_enabled(self) -> bool:
|
||||
"""Check if SSL is enabled."""
|
||||
if not self.config_content:
|
||||
return False
|
||||
return 'ssl_certificate' in self.config_content
|
||||
|
||||
def _extract_http_ports(self) -> List[int]:
|
||||
"""Extract HTTP listening ports."""
|
||||
if not self.config_content:
|
||||
return []
|
||||
pattern = r'listen\s+(\d+)'
|
||||
matches = re.findall(pattern, self.config_content)
|
||||
return sorted(list(set(int(p) for p in matches if int(p) < 1000)))
|
||||
|
||||
def _extract_https_ports(self) -> List[int]:
|
||||
"""Extract HTTPS listening ports."""
|
||||
if not self.config_content:
|
||||
return []
|
||||
pattern = r'listen\s+(\d+).*ssl'
|
||||
matches = re.findall(pattern, self.config_content)
|
||||
return sorted(list(set(int(p) for p in matches)))
|
||||
|
||||
def _extract_upstream_servers(self) -> List[str]:
|
||||
"""Extract upstream servers."""
|
||||
if not self.config_content:
|
||||
return []
|
||||
upstream_match = re.search(r'upstream\s+\w+\s*{([^}]+)}', self.config_content)
|
||||
if upstream_match:
|
||||
upstream_content = upstream_match.group(1)
|
||||
servers = re.findall(r'server\s+([^\s;]+)', upstream_content)
|
||||
return servers
|
||||
return []
|
||||
|
||||
def _extract_server_names(self) -> List[str]:
|
||||
"""Extract server names."""
|
||||
if not self.config_content:
|
||||
return []
|
||||
pattern = r'server_name\s+([^;]+);'
|
||||
matches = re.findall(pattern, self.config_content)
|
||||
result = []
|
||||
for match in matches:
|
||||
names = match.strip().split()
|
||||
result.extend(names)
|
||||
return result
|
||||
|
||||
def _extract_ssl_protocols(self) -> List[str]:
|
||||
"""Extract SSL protocols."""
|
||||
if not self.config_content:
|
||||
return []
|
||||
pattern = r'ssl_protocols\s+([^;]+);'
|
||||
match = re.search(pattern, self.config_content)
|
||||
if match:
|
||||
return match.group(1).strip().split()
|
||||
return []
|
||||
|
||||
def _extract_client_max_body_size(self) -> Optional[str]:
|
||||
"""Extract client max body size."""
|
||||
if not self.config_content:
|
||||
return None
|
||||
pattern = r'client_max_body_size\s+([^;]+);'
|
||||
match = re.search(pattern, self.config_content)
|
||||
return match.group(1).strip() if match else None
|
||||
|
||||
def _check_gzip_enabled(self) -> bool:
|
||||
"""Check if gzip is enabled."""
|
||||
if not self.config_content:
|
||||
return False
|
||||
return bool(re.search(r'gzip\s+on\s*;', self.config_content))
|
||||
|
||||
|
||||
def get_nginx_status() -> Dict[str, Any]:
|
||||
"""Get Nginx configuration status."""
|
||||
reader = NginxConfigReader()
|
||||
return reader.get_status()
|
||||
+281
-57
@@ -8,12 +8,25 @@ Admins use the "Build player files" admin page to:
|
||||
|
||||
The SSH deployment flow then ships this staged directory to player devices, so
|
||||
the version admins build here is exactly what gets deployed.
|
||||
|
||||
Performance note
|
||||
----------------
|
||||
The player repository is large (~200 MB) and a full clone takes ~90 s. Because
|
||||
the build runs inside an HTTP request, that would exceed gunicorn's worker
|
||||
timeout and the worker would be killed mid-clone, leaving a broken checkout.
|
||||
Two mitigations are used together:
|
||||
|
||||
* **Shallow clones** (``--depth 1``) — only the tip of the requested branch is
|
||||
fetched, which is all a deployment needs. Drastically reduces transfer size.
|
||||
* **Background execution** — the admin route starts the build in a daemon
|
||||
thread and the page polls for progress, so no worker ever blocks on git.
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import logging
|
||||
import threading
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
@@ -24,90 +37,154 @@ logger = logging.getLogger(__name__)
|
||||
# Metadata file name stored in the Flask instance folder.
|
||||
BUILD_META_FILENAME = 'player_build.json'
|
||||
|
||||
# Only the tip of the branch is needed to deploy a player, so history is not
|
||||
# fetched. Keeps the transfer small enough to avoid worker timeouts.
|
||||
CLONE_DEPTH = '1'
|
||||
|
||||
def _run_git(args, cwd=None, timeout=300) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(
|
||||
['git'] + args,
|
||||
cwd=cwd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
# Never let git wait for a human. Without this, a private/renamed repository
|
||||
# makes git block on a username prompt until the worker is killed.
|
||||
GIT_ENV = {
|
||||
'GIT_TERMINAL_PROMPT': '0', # never prompt for credentials
|
||||
'GIT_ASKPASS': 'true', # answer any credential request immediately
|
||||
'GIT_SSH_COMMAND': 'ssh -oBatchMode=yes -oStrictHostKeyChecking=accept-new',
|
||||
}
|
||||
|
||||
|
||||
def _git_env() -> Dict[str, str]:
|
||||
"""Environment for git subprocesses: inherit the process env plus our flags."""
|
||||
env = dict(os.environ)
|
||||
env.update(GIT_ENV)
|
||||
return env
|
||||
|
||||
|
||||
def _run_git(args, cwd=None, timeout=120) -> subprocess.CompletedProcess:
|
||||
"""Run a git command, never prompting for input.
|
||||
|
||||
Args:
|
||||
args: git arguments (without the leading 'git').
|
||||
cwd: working directory for the command.
|
||||
timeout: hard cap in seconds. Defaults to 120 to stay within a
|
||||
reasonable window even when running in the foreground.
|
||||
|
||||
Returns:
|
||||
The completed process. ``returncode`` is 124 on timeout so callers can
|
||||
distinguish a timeout from a normal failure.
|
||||
"""
|
||||
try:
|
||||
return subprocess.run(
|
||||
['git'] + args,
|
||||
cwd=cwd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
env=_git_env(),
|
||||
)
|
||||
except subprocess.TimeoutExpired as e:
|
||||
# Surface timeouts as a normal result so callers do not need try/except.
|
||||
out = e.stdout.decode() if isinstance(e.stdout, bytes) else (e.stdout or '')
|
||||
err = e.stderr.decode() if isinstance(e.stderr, bytes) else (e.stderr or '')
|
||||
return subprocess.CompletedProcess(
|
||||
args=['git'] + list(args), returncode=124,
|
||||
stdout=out, stderr=(err + f'\ngit {" ".join(args)} timed out after {timeout}s').strip(),
|
||||
)
|
||||
|
||||
|
||||
def get_short_head(player_code_dir: str) -> str:
|
||||
"""Return the short git commit of the staged code, or 'unknown'."""
|
||||
try:
|
||||
result = _run_git(['-C', player_code_dir, 'rev-parse', '--short', 'HEAD'], timeout=10)
|
||||
if result.returncode == 0:
|
||||
return result.stdout.strip()
|
||||
except Exception:
|
||||
pass
|
||||
result = _run_git(['-C', player_code_dir, 'rev-parse', '--short', 'HEAD'], timeout=10)
|
||||
if result.returncode == 0:
|
||||
return result.stdout.strip()
|
||||
return 'unknown'
|
||||
|
||||
|
||||
def is_valid_checkout(path: str) -> bool:
|
||||
"""True when *path* is a usable git checkout with a resolvable HEAD."""
|
||||
if not os.path.isdir(os.path.join(path, '.git')):
|
||||
return False
|
||||
return get_short_head(path) != 'unknown'
|
||||
|
||||
|
||||
def _clone(path: str, repo_url: str, branch: str) -> subprocess.CompletedProcess:
|
||||
"""Shallow-clone a single branch into *path*."""
|
||||
return _run_git([
|
||||
'clone', '--depth', CLONE_DEPTH, '--single-branch',
|
||||
'--branch', branch, repo_url, path,
|
||||
], timeout=600)
|
||||
|
||||
|
||||
def build_player_files(player_code_dir: str, repo_url: str, branch: str = 'main') -> Dict[str, Any]:
|
||||
"""Clone or refresh the player source into ``player_code_dir``.
|
||||
|
||||
If the directory is already a git checkout of ``repo_url`` it is updated in
|
||||
place (fetch + hard reset to the chosen branch). Otherwise it is cloned
|
||||
fresh (an existing non-git directory is replaced).
|
||||
Uses a **shallow single-branch clone/update** so only the tip of the wanted
|
||||
branch is transferred. If the directory is a usable checkout it is updated
|
||||
(fetch + hard reset to the branch). A directory that exists but is NOT a
|
||||
usable checkout — e.g. left behind by an interrupted clone — is removed and
|
||||
re-cloned, since updating it can never work.
|
||||
|
||||
Returns a dict: ``success`` (bool), ``message`` (str), ``version`` (str),
|
||||
``branch`` (str).
|
||||
Args:
|
||||
player_code_dir: Destination directory for the staged player code.
|
||||
repo_url: Git repository to pull from.
|
||||
branch: Branch to stage.
|
||||
|
||||
Returns:
|
||||
``{'success': bool, 'message': str, 'version': str|None, 'branch': str}``
|
||||
"""
|
||||
branch = (branch or 'main').strip()
|
||||
repo_url = (repo_url or '').strip()
|
||||
|
||||
def fail(message: str) -> Dict[str, Any]:
|
||||
return {'success': False, 'message': message,
|
||||
'version': get_short_head(player_code_dir), 'branch': branch}
|
||||
|
||||
if not repo_url:
|
||||
return {'success': False, 'message': 'Repository URL is required.', 'version': None, 'branch': branch}
|
||||
return {'success': False, 'message': 'Repository URL is required.',
|
||||
'version': None, 'branch': branch}
|
||||
|
||||
try:
|
||||
git_dir = os.path.join(player_code_dir, '.git')
|
||||
is_git_repo = os.path.isdir(git_dir)
|
||||
usable = is_valid_checkout(player_code_dir)
|
||||
|
||||
if is_git_repo:
|
||||
# Update existing checkout in place.
|
||||
fetch = _run_git(['-C', player_code_dir, 'fetch', '--prune', 'origin'])
|
||||
if usable:
|
||||
# Update in place. Depth 1 keeps the update cheap; fetch by ref so
|
||||
# it works on a shallow clone.
|
||||
fetch = _run_git(
|
||||
['-C', player_code_dir, 'fetch', '--depth', CLONE_DEPTH,
|
||||
'--prune', 'origin', branch])
|
||||
if fetch.returncode != 0:
|
||||
return {
|
||||
'success': False,
|
||||
'message': f'git fetch failed: {fetch.stderr.strip() or fetch.stdout.strip()}',
|
||||
'version': get_short_head(player_code_dir),
|
||||
'branch': branch,
|
||||
}
|
||||
return fail(f'git fetch failed: {fetch.stderr.strip() or fetch.stdout.strip()}')
|
||||
|
||||
# Point origin at the requested URL in case it changed.
|
||||
_run_git(['-C', player_code_dir, 'remote', 'set-url', 'origin', repo_url])
|
||||
|
||||
checkout = _run_git(['-C', player_code_dir, 'checkout', branch])
|
||||
if checkout.returncode != 0:
|
||||
return {
|
||||
'success': False,
|
||||
'message': f'git checkout {branch} failed: {checkout.stderr.strip()}',
|
||||
'version': get_short_head(player_code_dir),
|
||||
'branch': branch,
|
||||
}
|
||||
return fail(f'git checkout {branch} failed: {checkout.stderr.strip()}')
|
||||
|
||||
reset = _run_git(['-C', player_code_dir, 'reset', '--hard', f'origin/{branch}'])
|
||||
if reset.returncode != 0:
|
||||
return {
|
||||
'success': False,
|
||||
'message': f'git reset failed: {reset.stderr.strip()}',
|
||||
'version': get_short_head(player_code_dir),
|
||||
'branch': branch,
|
||||
}
|
||||
return fail(f'git reset failed: {reset.stderr.strip()}')
|
||||
|
||||
action = 'Updated'
|
||||
else:
|
||||
# Fresh clone. Replace any existing (non-git) directory.
|
||||
# Fresh clone. A previous attempt may have left a partial directory
|
||||
# (e.g. killed mid-clone) — it must go, or the clone will fail with
|
||||
# "destination path already exists and is not an empty directory".
|
||||
parent = os.path.dirname(player_code_dir.rstrip('/'))
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
if parent:
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
if os.path.exists(player_code_dir):
|
||||
shutil.rmtree(player_code_dir)
|
||||
clone = _run_git(['clone', '--branch', branch, repo_url, player_code_dir])
|
||||
logger.info('Removing unusable directory before clone: %s', player_code_dir)
|
||||
shutil.rmtree(player_code_dir, ignore_errors=True)
|
||||
|
||||
clone = _clone(player_code_dir, repo_url, branch)
|
||||
if clone.returncode != 0:
|
||||
return {
|
||||
'success': False,
|
||||
'message': f'git clone failed: {clone.stderr.strip() or clone.stdout.strip()}',
|
||||
'version': None,
|
||||
'branch': branch,
|
||||
}
|
||||
# Do not leave a half-written directory behind.
|
||||
shutil.rmtree(player_code_dir, ignore_errors=True)
|
||||
detail = clone.stderr.strip() or clone.stdout.strip()
|
||||
if clone.returncode == 124 or 'timed out' in detail:
|
||||
return fail(f'git clone timed out. The repository may be very '
|
||||
f'large or unreachable: {detail}')
|
||||
return fail(f'git clone failed: {detail}')
|
||||
|
||||
action = 'Cloned'
|
||||
|
||||
version = get_short_head(player_code_dir)
|
||||
@@ -118,11 +195,15 @@ def build_player_files(player_code_dir: str, repo_url: str, branch: str = 'main'
|
||||
'version': version,
|
||||
'branch': branch,
|
||||
}
|
||||
except subprocess.TimeoutExpired:
|
||||
return {'success': False, 'message': 'Git operation timed out.', 'version': None, 'branch': branch}
|
||||
except Exception as e:
|
||||
except Exception as e: # noqa: BLE001 - surface any failure
|
||||
logger.exception('build_player_files failed')
|
||||
return {'success': False, 'message': f'Build failed: {str(e)}', 'version': None, 'branch': branch}
|
||||
# Never leave a broken checkout behind for the next attempt.
|
||||
try:
|
||||
if not is_valid_checkout(player_code_dir):
|
||||
shutil.rmtree(player_code_dir, ignore_errors=True)
|
||||
except Exception:
|
||||
pass
|
||||
return fail(f'Build failed: {str(e)}')
|
||||
|
||||
|
||||
def write_base_config(
|
||||
@@ -221,3 +302,146 @@ def make_build_record(repo_url, branch, server_ip, port, use_https, verify_ssl,
|
||||
'built_at': datetime.utcnow().isoformat(timespec='seconds') + 'Z',
|
||||
'built_by': built_by,
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Background builds
|
||||
#
|
||||
# A full clone/refresh takes far longer than gunicorn's worker timeout, so the
|
||||
# build must not run inside the request. The admin route starts it here and the
|
||||
# page polls `build_state()` for progress.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Serialises writes to _build_state between the request thread and the worker.
|
||||
_build_lock = threading.Lock()
|
||||
|
||||
# Coarse progress for the admin UI. 'state' is one of:
|
||||
# idle | running | success | error
|
||||
_build_state: Dict[str, Any] = {'state': 'idle'}
|
||||
|
||||
|
||||
def get_build_state() -> Dict[str, Any]:
|
||||
"""Return a snapshot of the current/last build for the admin UI."""
|
||||
with _build_lock:
|
||||
return dict(_build_state)
|
||||
|
||||
|
||||
def is_build_running() -> bool:
|
||||
"""True while a build is in progress."""
|
||||
with _build_lock:
|
||||
return _build_state.get('state') == 'running'
|
||||
|
||||
|
||||
def _set_build_state(**fields: Any) -> None:
|
||||
with _build_lock:
|
||||
_build_state.update(fields)
|
||||
|
||||
|
||||
def _run_build_job(app, player_code_dir: str, repo_url: str, branch: str,
|
||||
config_payload: Optional[Dict[str, Any]],
|
||||
meta_path: str, built_by: str) -> None:
|
||||
"""Worker body: build files, optionally write config, then persist settings.
|
||||
|
||||
Runs in a daemon thread with its own Flask app context so it is independent
|
||||
of the request/response cycle that triggered it.
|
||||
"""
|
||||
started = datetime.utcnow()
|
||||
try:
|
||||
_set_build_state(state='running', step='Fetching player source…',
|
||||
started_at=started.isoformat(timespec='seconds') + 'Z',
|
||||
message='', version=None)
|
||||
|
||||
result = build_player_files(player_code_dir, repo_url, branch)
|
||||
version = result.get('version')
|
||||
|
||||
if not result['success']:
|
||||
_set_build_state(state='error', step='', message=result['message'],
|
||||
version=version,
|
||||
finished_at=datetime.utcnow().isoformat(timespec='seconds') + 'Z')
|
||||
logger.error('Background player build failed: %s', result['message'])
|
||||
return
|
||||
|
||||
# Optional step 2: write the base config.
|
||||
if config_payload:
|
||||
_set_build_state(step='Writing player config…')
|
||||
cfg = write_base_config(player_code_dir=player_code_dir, **config_payload)
|
||||
if not cfg['success']:
|
||||
_set_build_state(state='error', step='', message=cfg['message'],
|
||||
version=version,
|
||||
finished_at=datetime.utcnow().isoformat(timespec='seconds') + 'Z')
|
||||
logger.error('Background player config write failed: %s', cfg['message'])
|
||||
return
|
||||
result = {**result, 'message': f"{result['message']} {cfg['message']}"}
|
||||
|
||||
if version is None:
|
||||
version = get_short_head(player_code_dir)
|
||||
|
||||
save_build_settings(
|
||||
meta_path,
|
||||
make_build_record(
|
||||
repo_url=repo_url, branch=branch,
|
||||
server_ip=(config_payload or {}).get('server_ip', ''),
|
||||
port=(config_payload or {}).get('port', ''),
|
||||
use_https=(config_payload or {}).get('use_https', False),
|
||||
verify_ssl=(config_payload or {}).get('verify_ssl', False),
|
||||
orientation=(config_payload or {}).get('orientation', 'Landscape'),
|
||||
max_resolution=(config_payload or {}).get('max_resolution', '1920x1080'),
|
||||
version=version, built_by=built_by,
|
||||
),
|
||||
)
|
||||
|
||||
_set_build_state(state='success', step='', message=result['message'],
|
||||
version=version,
|
||||
finished_at=datetime.utcnow().isoformat(timespec='seconds') + 'Z')
|
||||
logger.info('Background player build complete (version %s)', version)
|
||||
except Exception as e: # noqa: BLE001 - never kill the thread silently
|
||||
logger.exception('Background player build crashed')
|
||||
_set_build_state(state='error', step='', message=f'Build failed: {e}',
|
||||
finished_at=datetime.utcnow().isoformat(timespec='seconds') + 'Z')
|
||||
|
||||
|
||||
def start_background_build(player_code_dir: str, repo_url: str, branch: str,
|
||||
config_payload: Optional[Dict[str, Any]],
|
||||
meta_path: str, built_by: str) -> bool:
|
||||
"""Start a player build in a daemon thread.
|
||||
|
||||
Args:
|
||||
player_code_dir: Where to stage the player source.
|
||||
repo_url: Git repository URL.
|
||||
branch: Branch to stage.
|
||||
config_payload: Keyword args for :func:`write_base_config`, or None to
|
||||
skip writing the config.
|
||||
meta_path: Where to persist the build record.
|
||||
built_by: Username shown in the UI/logs.
|
||||
|
||||
Returns:
|
||||
False if a build is already running (callers should tell the user),
|
||||
True if a new build was started.
|
||||
|
||||
Raises:
|
||||
RuntimeError: if called with no Flask application context — the worker
|
||||
thread needs a real app object to push its own context.
|
||||
"""
|
||||
from flask import current_app
|
||||
|
||||
if is_build_running():
|
||||
return False
|
||||
|
||||
# Capture the real app object now. `current_app` resolves inside either a
|
||||
# request or a plain application context; the worker thread pushes its own
|
||||
# context later, since the caller's context is gone by then.
|
||||
app = current_app._get_current_object()
|
||||
|
||||
_set_build_state(state='running', step='Starting…', message='', version=None,
|
||||
started_at=datetime.utcnow().isoformat(timespec='seconds') + 'Z',
|
||||
finished_at=None, built_by=built_by,
|
||||
repo_url=repo_url, branch=branch)
|
||||
|
||||
thread = threading.Thread(
|
||||
target=_run_build_job,
|
||||
args=(app, player_code_dir, repo_url, branch, config_payload, meta_path, built_by),
|
||||
name='player-build',
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user