feat: execution failure reports, auto-printer for WMT, UTC timezone fix for all timestamps
This commit is contained in:
@@ -29,6 +29,9 @@ def create_app(config_name=None):
|
||||
# Register blueprints
|
||||
_register_blueprints(app)
|
||||
|
||||
# Register template filters
|
||||
_register_template_filters(app)
|
||||
|
||||
# Register error handlers
|
||||
_register_error_handlers(app)
|
||||
|
||||
@@ -104,6 +107,28 @@ def _register_blueprints(app):
|
||||
from app.api.logs import submit_log
|
||||
return submit_log()
|
||||
|
||||
|
||||
def _register_template_filters(app):
|
||||
"""Register custom Jinja2 template filters."""
|
||||
import calendar
|
||||
from datetime import datetime as _dt
|
||||
|
||||
@app.template_filter('local_dt')
|
||||
def local_dt_filter(value, fmt='%Y-%m-%d %H:%M'):
|
||||
"""Convert a naive UTC datetime to server local time and format it.
|
||||
Usage: {{ some_utc_datetime | local_dt }}
|
||||
{{ some_utc_datetime | local_dt('%Y-%m-%d %H:%M:%S') }}
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
# calendar.timegm treats the timetuple as UTC → returns POSIX timestamp
|
||||
# datetime.fromtimestamp then converts to local time using the system timezone
|
||||
ts = calendar.timegm(value.timetuple())
|
||||
return _dt.fromtimestamp(ts).strftime(fmt)
|
||||
except Exception:
|
||||
return value.strftime(fmt)
|
||||
|
||||
def _register_error_handlers(app):
|
||||
"""Register error handlers"""
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
from app.services.ansible_service import AnsibleService
|
||||
from app.models import Device, AnsibleExecution
|
||||
from app.models import Device, AnsibleExecution, ExecutionFailureReport
|
||||
from config.database_config import get_db
|
||||
import logging
|
||||
|
||||
@@ -148,7 +148,12 @@ def list_playbooks():
|
||||
'name': 'restart_service',
|
||||
'description': 'Restart monitoring services on devices',
|
||||
'builtin': True
|
||||
}
|
||||
},
|
||||
{
|
||||
'name': 'distribute_ssh_keys',
|
||||
'description': 'Push server public key to devices using password auth',
|
||||
'builtin': True
|
||||
},
|
||||
]
|
||||
|
||||
return jsonify({
|
||||
@@ -353,6 +358,88 @@ def test_ssh_connectivity():
|
||||
'success': False
|
||||
}), 500
|
||||
|
||||
|
||||
@ansible_bp.route('/ssh/test-password', methods=['POST'])
|
||||
def test_password_auth():
|
||||
"""
|
||||
Test password-only SSH authentication to a single device.
|
||||
Use this to verify the configured password is correct before deploying keys.
|
||||
|
||||
Expected JSON:
|
||||
{
|
||||
"device_ip": "10.76.157.145",
|
||||
"password": "raspberry", # optional — uses saved setting if omitted
|
||||
"username": "pi", # optional
|
||||
"port": 22 # optional
|
||||
}
|
||||
"""
|
||||
try:
|
||||
data = request.get_json() or {}
|
||||
device_ip = (data.get('device_ip') or '').strip()
|
||||
if not device_ip:
|
||||
return jsonify({'success': False, 'error': 'device_ip is required'}), 400
|
||||
|
||||
# Use provided password, fall back to saved setting
|
||||
password = data.get('password') or ansible_service.load_settings().get('ssh_fallback_password', '')
|
||||
if not password:
|
||||
return jsonify({'success': False,
|
||||
'error': 'No password provided and none saved in SSH Settings'}), 400
|
||||
|
||||
username = data.get('username', 'pi')
|
||||
port = int(data.get('port', 22))
|
||||
|
||||
result = ansible_service.test_password_auth(device_ip, password, username, port)
|
||||
status = 200 if result.get('success') else 400
|
||||
return jsonify(result), status
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error testing password auth: {e}")
|
||||
return jsonify({'success': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@ansible_bp.route('/ssh/distribute-keys', methods=['POST'])
|
||||
def distribute_ssh_keys():
|
||||
"""
|
||||
Push the server's public SSH key to all (or selected) devices using password auth.
|
||||
After this completes, all other playbooks can use key-based authentication.
|
||||
|
||||
Optional JSON body:
|
||||
{
|
||||
"limit_hosts": ["RPI-ABC1", "RPI-ABC2"] # omit to target all devices
|
||||
}
|
||||
"""
|
||||
try:
|
||||
settings = ansible_service.load_settings()
|
||||
if not settings.get('ssh_fallback_password'):
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': 'No SSH password configured. Set it in SSH Settings before distributing keys.'
|
||||
}), 400
|
||||
|
||||
# Make sure the public key exists
|
||||
public_key_path = ansible_service.ssh_key_path.with_suffix('.pub')
|
||||
if not public_key_path.exists():
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'error': 'Public key not found at ansible/ssh_keys/app_key.pub. Generate SSH keys first.'
|
||||
}), 400
|
||||
|
||||
data = request.get_json() or {}
|
||||
limit_hosts = data.get('limit_hosts') or None
|
||||
|
||||
ansible_service.create_distribute_ssh_keys_playbook()
|
||||
result = ansible_service.execute_playbook_async(
|
||||
playbook_name='distribute_ssh_keys',
|
||||
limit_hosts=limit_hosts,
|
||||
force_password_auth=True,
|
||||
)
|
||||
return jsonify(result), 200 if result.get('success') else 500
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error distributing SSH keys: {e}")
|
||||
return jsonify({'success': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@ansible_bp.route('/ssh/keys/setup', methods=['POST'])
|
||||
def setup_ssh_keys():
|
||||
"""Setup SSH keys for Ansible authentication"""
|
||||
@@ -451,4 +538,119 @@ def service_restart_callback():
|
||||
return jsonify({'success': True})
|
||||
except Exception as e:
|
||||
logging.error(f"Error in service restart callback: {e}")
|
||||
return jsonify({'error': str(e)}), 500
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
# ── Failure Reports ────────────────────────────────────────────────────── #
|
||||
|
||||
@ansible_bp.route('/failure-reports', methods=['GET'])
|
||||
def list_failure_reports():
|
||||
"""Return all saved execution failure reports, newest first."""
|
||||
try:
|
||||
with get_db().get_session() as session:
|
||||
reports = session.query(ExecutionFailureReport)\
|
||||
.order_by(ExecutionFailureReport.saved_at.desc())\
|
||||
.all()
|
||||
return jsonify({'success': True, 'reports': [r.to_dict() for r in reports]})
|
||||
except Exception as e:
|
||||
logging.error(f"Error listing failure reports: {e}")
|
||||
return jsonify({'success': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@ansible_bp.route('/failure-reports', methods=['POST'])
|
||||
def save_failure_report():
|
||||
"""
|
||||
Save a failure report for an execution.
|
||||
Parses the PLAY RECAP from the execution log to extract per-host failure reasons.
|
||||
|
||||
Expected JSON:
|
||||
{
|
||||
"execution_id": "uuid",
|
||||
"note": "optional free-text note" # optional
|
||||
}
|
||||
"""
|
||||
import re
|
||||
try:
|
||||
data = request.get_json() or {}
|
||||
execution_id = (data.get('execution_id') or '').strip()
|
||||
if not execution_id:
|
||||
return jsonify({'success': False, 'error': 'execution_id is required'}), 400
|
||||
|
||||
with get_db().get_session() as session:
|
||||
from app.models import PlaybookExecution
|
||||
execution = session.query(PlaybookExecution).filter_by(
|
||||
execution_id=execution_id
|
||||
).first()
|
||||
if not execution:
|
||||
return jsonify({'success': False, 'error': 'Execution not found'}), 404
|
||||
|
||||
# Parse PLAY RECAP for per-host stats
|
||||
log_text = execution.stdout_log or ''
|
||||
recap_re = re.compile(
|
||||
r'^(\S.*?)\s*:\s*ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)',
|
||||
re.MULTILINE
|
||||
)
|
||||
failed_hosts = []
|
||||
failed_count = 0
|
||||
unreachable_count = 0
|
||||
for m in recap_re.finditer(log_text):
|
||||
hostname = m.group(1).strip()
|
||||
unreachable = int(m.group(4))
|
||||
failed = int(m.group(5))
|
||||
if unreachable > 0:
|
||||
failed_hosts.append({'hostname': hostname, 'status': 'unreachable',
|
||||
'reason': 'Host unreachable via SSH'})
|
||||
unreachable_count += 1
|
||||
elif failed > 0:
|
||||
failed_hosts.append({'hostname': hostname, 'status': 'failed',
|
||||
'reason': f'{failed} task(s) failed'})
|
||||
failed_count += 1
|
||||
|
||||
if not failed_hosts:
|
||||
return jsonify({'success': False,
|
||||
'error': 'No failed or unreachable hosts found in this execution'}), 400
|
||||
|
||||
# Avoid duplicate reports for the same execution
|
||||
existing = session.query(ExecutionFailureReport).filter_by(
|
||||
execution_id=execution_id
|
||||
).first()
|
||||
if existing:
|
||||
return jsonify({'success': False,
|
||||
'error': 'A report for this execution already exists',
|
||||
'report_id': existing.id}), 409
|
||||
|
||||
report = ExecutionFailureReport(
|
||||
execution_id=execution_id,
|
||||
playbook_name=execution.playbook_name,
|
||||
saved_at=datetime.utcnow(),
|
||||
failed_count=failed_count,
|
||||
unreachable_count=unreachable_count,
|
||||
failed_hosts=json.dumps(failed_hosts),
|
||||
note=data.get('note', ''),
|
||||
)
|
||||
session.add(report)
|
||||
session.flush()
|
||||
report_id = report.id
|
||||
|
||||
return jsonify({'success': True, 'report_id': report_id,
|
||||
'failed_count': failed_count,
|
||||
'unreachable_count': unreachable_count}), 201
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error saving failure report: {e}")
|
||||
return jsonify({'success': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@ansible_bp.route('/failure-reports/<int:report_id>', methods=['DELETE'])
|
||||
def delete_failure_report(report_id):
|
||||
"""Delete a saved failure report."""
|
||||
try:
|
||||
with get_db().get_session() as session:
|
||||
report = session.query(ExecutionFailureReport).get(report_id)
|
||||
if not report:
|
||||
return jsonify({'success': False, 'error': 'Report not found'}), 404
|
||||
session.delete(report)
|
||||
return jsonify({'success': True})
|
||||
except Exception as e:
|
||||
logging.error(f"Error deleting failure report {report_id}: {e}")
|
||||
return jsonify({'success': False, 'error': str(e)}), 500
|
||||
@@ -26,7 +26,7 @@ __all__ = [
|
||||
'Base', 'Device', 'MessageTemplate', 'LogEntry', 'FileUpload',
|
||||
'AnsibleExecution', 'SystemStats', 'InventoryGroup', 'PlaybookExecution',
|
||||
'PlaybookHostResult', 'ExecutionQueue', 'device_inventory_association',
|
||||
'WMTGlobalConfig', 'WMTUpdateRequest',
|
||||
'WMTGlobalConfig', 'WMTUpdateRequest', 'ExecutionFailureReport',
|
||||
]
|
||||
|
||||
class Device(Base):
|
||||
@@ -626,4 +626,48 @@ class WMTUpdateRequest(Base):
|
||||
}
|
||||
|
||||
def __repr__(self):
|
||||
return f"<WMTUpdateRequest(mac='{self.mac_address}', status='{self.status}')>"
|
||||
return f"<WMTUpdateRequest(mac='{self.mac_address}', status='{self.status}')>"
|
||||
|
||||
|
||||
class ExecutionFailureReport(Base):
|
||||
"""Saved report of failed/unreachable hosts from a playbook execution."""
|
||||
__tablename__ = 'execution_failure_reports'
|
||||
|
||||
id = Column(Integer, primary_key=True)
|
||||
execution_id = Column(String(36), nullable=False, index=True)
|
||||
playbook_name = Column(String(255), nullable=False)
|
||||
saved_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
|
||||
# Counts
|
||||
failed_count = Column(Integer, default=0)
|
||||
unreachable_count = Column(Integer, default=0)
|
||||
|
||||
# JSON list of {hostname, status, reason} objects
|
||||
failed_hosts = Column(Text, nullable=False, default='[]')
|
||||
|
||||
# Optional note added by user when saving
|
||||
note = Column(Text)
|
||||
|
||||
@property
|
||||
def hosts_list(self):
|
||||
try:
|
||||
return json.loads(self.failed_hosts)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
'id': self.id,
|
||||
'execution_id': self.execution_id,
|
||||
'playbook_name': self.playbook_name,
|
||||
'saved_at': self.saved_at.isoformat() if self.saved_at else None,
|
||||
'failed_count': self.failed_count,
|
||||
'unreachable_count': self.unreachable_count,
|
||||
'failed_hosts': self.hosts_list,
|
||||
'note': self.note,
|
||||
}
|
||||
|
||||
def __repr__(self):
|
||||
return (f"<ExecutionFailureReport(execution_id='{self.execution_id}', "
|
||||
f"playbook='{self.playbook_name}', failed={self.failed_count}, "
|
||||
f"unreachable={self.unreachable_count})>")
|
||||
@@ -23,6 +23,7 @@ class AnsibleService:
|
||||
SETTINGS_FILE = Path("data/ansible_settings.json")
|
||||
DEFAULT_SETTINGS = {
|
||||
"ssh_fallback_password": "raspberry",
|
||||
"use_password_auth": False,
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
@@ -30,13 +31,16 @@ class AnsibleService:
|
||||
self.ansible_dir = Path("ansible")
|
||||
self.inventory_file = self.ansible_dir / "inventory" / "dynamic_inventory.yaml"
|
||||
self.playbook_dir = self.ansible_dir / "playbooks"
|
||||
self.ssh_key_path = Path.home() / ".ssh" / "ansible_key"
|
||||
self.ssh_keys_dir = self.ansible_dir / "ssh_keys"
|
||||
self.ssh_key_path = self.ssh_keys_dir / "app_key"
|
||||
self.ansible_cfg_path = self.ansible_dir / "ansible.cfg"
|
||||
|
||||
# Ensure directories exist
|
||||
self.ansible_dir.mkdir(exist_ok=True)
|
||||
(self.ansible_dir / "inventory").mkdir(exist_ok=True)
|
||||
(self.ansible_dir / "playbooks").mkdir(exist_ok=True)
|
||||
(self.ansible_dir / "roles").mkdir(exist_ok=True)
|
||||
self.ssh_keys_dir.mkdir(mode=0o700, exist_ok=True)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Settings helpers #
|
||||
@@ -136,12 +140,24 @@ class AnsibleService:
|
||||
'ansible_host': '127.0.0.1'
|
||||
}
|
||||
else:
|
||||
hvars = {
|
||||
'ansible_host': device.device_ip,
|
||||
'ansible_user': 'pi',
|
||||
'ansible_ssh_private_key_file': str(self.ssh_key_path),
|
||||
'ansible_ssh_common_args': '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'
|
||||
}
|
||||
settings = self.load_settings()
|
||||
use_password = settings.get('use_password_auth', False)
|
||||
ssh_password = settings.get('ssh_fallback_password', '')
|
||||
if use_password and ssh_password:
|
||||
hvars = {
|
||||
'ansible_host': device.device_ip,
|
||||
'ansible_user': 'pi',
|
||||
'ansible_password': ssh_password,
|
||||
'ansible_become_password': ssh_password,
|
||||
'ansible_ssh_common_args': '-o PubkeyAuthentication=no -o PreferredAuthentications=password -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'
|
||||
}
|
||||
else:
|
||||
hvars = {
|
||||
'ansible_host': device.device_ip,
|
||||
'ansible_user': 'pi',
|
||||
'ansible_ssh_private_key_file': str(self.ssh_key_path.resolve()),
|
||||
'ansible_ssh_common_args': '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'
|
||||
}
|
||||
children['monitoring_devices']['hosts'][device.hostname] = hvars
|
||||
synced += 1
|
||||
self._write_inventory(data)
|
||||
@@ -249,7 +265,7 @@ class AnsibleService:
|
||||
'name': 'Update monitoring devices',
|
||||
'hosts': 'all',
|
||||
'become': True,
|
||||
'gather_facts': True,
|
||||
'gather_facts': False,
|
||||
'tasks': [
|
||||
{
|
||||
'name': 'Update apt cache',
|
||||
@@ -268,40 +284,24 @@ class AnsibleService:
|
||||
'register': 'upgrade_result'
|
||||
},
|
||||
{
|
||||
'name': 'Restart device if required',
|
||||
'reboot': {
|
||||
'reboot_timeout': 600
|
||||
},
|
||||
'when': 'upgrade_result.changed'
|
||||
},
|
||||
{
|
||||
'name': 'Check service status',
|
||||
'systemd': {
|
||||
'name': 'prezenta.service',
|
||||
'state': 'started'
|
||||
'name': 'Show upgrade result',
|
||||
'debug': {
|
||||
'msg': '{{ upgrade_result.stdout_lines }}'
|
||||
}
|
||||
},
|
||||
{
|
||||
'name': 'Report update completion',
|
||||
'uri': {
|
||||
'url': 'http://{{ ansible_controller_ip }}/api/update_complete',
|
||||
'method': 'POST',
|
||||
'body_format': 'json',
|
||||
'body': {
|
||||
'hostname': '{{ inventory_hostname }}',
|
||||
'device_ip': '{{ ansible_host }}',
|
||||
'status': 'completed',
|
||||
'packages_updated': '{{ upgrade_result.stdout_lines | length }}'
|
||||
}
|
||||
'name': 'Clean up apt cache',
|
||||
'apt': {
|
||||
'autoclean': True
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
playbook_path = self.playbook_dir / "update_devices.yml"
|
||||
with open(playbook_path, 'w') as f:
|
||||
yaml.dump([playbook_content], f, default_flow_style=False)
|
||||
|
||||
|
||||
return str(playbook_path)
|
||||
|
||||
def create_restart_service_playbook(self) -> str:
|
||||
@@ -390,6 +390,17 @@ class AnsibleService:
|
||||
# Add extra variables
|
||||
if extra_vars:
|
||||
cmd.extend(['--extra-vars', json.dumps(extra_vars)])
|
||||
|
||||
# Inject password auth vars if enabled (overrides per-host inventory vars)
|
||||
settings = self.load_settings()
|
||||
if settings.get('use_password_auth') and settings.get('ssh_fallback_password'):
|
||||
pwd = settings['ssh_fallback_password']
|
||||
cmd.extend(['--extra-vars', json.dumps({
|
||||
'ansible_password': pwd,
|
||||
'ansible_become_password': pwd,
|
||||
'ansible_ssh_private_key_file': '',
|
||||
'ansible_ssh_common_args': '-o PubkeyAuthentication=no -o PreferredAuthentications=password -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'
|
||||
})])
|
||||
|
||||
# Create enhanced execution record using new model
|
||||
execution_id = str(uuid.uuid4())
|
||||
@@ -416,12 +427,19 @@ class AnsibleService:
|
||||
with tempfile.NamedTemporaryFile(mode='w+', suffix='.log', delete=False) as log_file:
|
||||
log_file_path = log_file.name
|
||||
|
||||
env = os.environ.copy()
|
||||
env['PYTHONUNBUFFERED'] = '1'
|
||||
env['ANSIBLE_FORCE_COLOR'] = '0'
|
||||
env['ANSIBLE_NOCOLOR'] = '1'
|
||||
env['ANSIBLE_CONFIG'] = str(self.ansible_cfg_path.resolve())
|
||||
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
cwd=str(self.ansible_dir)
|
||||
cwd=str(self.ansible_dir),
|
||||
env=env,
|
||||
)
|
||||
|
||||
stdout, stderr = process.communicate()
|
||||
@@ -435,11 +453,12 @@ class AnsibleService:
|
||||
execution.stderr_log = stderr
|
||||
execution.ansible_log_file = log_file_path
|
||||
|
||||
# Always parse recap stats regardless of exit code —
|
||||
# Ansible exits non-zero when any host fails/is unreachable.
|
||||
self._parse_ansible_results_enhanced(execution, stdout)
|
||||
if process.returncode == 0:
|
||||
execution.status = 'completed'
|
||||
execution.summary_message = 'Playbook executed successfully'
|
||||
# Parse stdout for success/failure counts
|
||||
self._parse_ansible_results_enhanced(execution, stdout)
|
||||
else:
|
||||
execution.status = 'failed'
|
||||
execution.summary_message = f'Playbook failed with exit code {process.returncode}'
|
||||
@@ -474,10 +493,14 @@ class AnsibleService:
|
||||
|
||||
def execute_playbook_async(self, playbook_name: str, limit_hosts: List[str] = None,
|
||||
extra_vars: Dict = None, priority: int = 5,
|
||||
max_retries: int = 0) -> Dict:
|
||||
max_retries: int = 0,
|
||||
force_password_auth: bool = False) -> Dict:
|
||||
"""
|
||||
Start a playbook in a background thread.
|
||||
Returns immediately with the execution_id so the caller can poll /live.
|
||||
force_password_auth=True overrides the use_password_auth setting and always
|
||||
injects password vars — used by distribute_ssh_keys which must run before
|
||||
keys are deployed.
|
||||
"""
|
||||
try:
|
||||
self.generate_dynamic_inventory()
|
||||
@@ -498,6 +521,17 @@ class AnsibleService:
|
||||
# Pass all extra vars as a single JSON string to avoid value-quoting issues
|
||||
cmd.extend(['--extra-vars', json.dumps(extra_vars)])
|
||||
|
||||
# Inject password auth vars if enabled OR forced
|
||||
settings = self.load_settings()
|
||||
if (force_password_auth or settings.get('use_password_auth')) and settings.get('ssh_fallback_password'):
|
||||
pwd = settings['ssh_fallback_password']
|
||||
cmd.extend(['--extra-vars', json.dumps({
|
||||
'ansible_password': pwd,
|
||||
'ansible_become_password': pwd,
|
||||
'ansible_ssh_private_key_file': '',
|
||||
'ansible_ssh_common_args': '-o PubkeyAuthentication=no -o PreferredAuthentications=password -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'
|
||||
})])
|
||||
|
||||
# Create a persistent log file (NOT deleted on close)
|
||||
log_fd, log_file_path = tempfile.mkstemp(suffix='.log', prefix='ansible_')
|
||||
os.close(log_fd)
|
||||
@@ -546,6 +580,7 @@ class AnsibleService:
|
||||
env['PYTHONUNBUFFERED'] = '1'
|
||||
env['ANSIBLE_FORCE_COLOR'] = '0'
|
||||
env['ANSIBLE_NOCOLOR'] = '1'
|
||||
env['ANSIBLE_CONFIG'] = str(self.ansible_cfg_path.resolve())
|
||||
|
||||
process = subprocess.Popen(
|
||||
cmd,
|
||||
@@ -586,10 +621,12 @@ class AnsibleService:
|
||||
execution.completed_at = datetime.utcnow()
|
||||
execution.exit_code = process.returncode
|
||||
execution.stdout_log = full_output
|
||||
# Always parse recap stats regardless of exit code —
|
||||
# Ansible exits non-zero when any host fails/is unreachable.
|
||||
self._parse_ansible_results_enhanced(execution, full_output)
|
||||
if process.returncode == 0:
|
||||
execution.status = 'completed'
|
||||
execution.summary_message = 'Playbook executed successfully'
|
||||
self._parse_ansible_results_enhanced(execution, full_output)
|
||||
else:
|
||||
execution.status = 'failed'
|
||||
execution.summary_message = f'Playbook failed (exit {process.returncode})'
|
||||
@@ -653,62 +690,62 @@ class AnsibleService:
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
def _parse_ansible_results_enhanced(self, execution: PlaybookExecution, output: str):
|
||||
"""Parse Ansible output for enhanced result statistics"""
|
||||
lines = output.split('\n')
|
||||
"""Parse Ansible PLAY RECAP output for result statistics."""
|
||||
import re
|
||||
successful_hosts = 0
|
||||
failed_hosts = 0
|
||||
unreachable_hosts = 0
|
||||
skipped_hosts = 0
|
||||
changed_hosts = 0
|
||||
|
||||
for line in lines:
|
||||
if 'ok=' in line and 'changed=' in line:
|
||||
# Parse line like: "host1: ok=4 changed=2 unreachable=0 failed=0"
|
||||
try:
|
||||
if 'failed=0' in line:
|
||||
successful_hosts += 1
|
||||
else:
|
||||
failed_count = int(line.split('failed=')[1].split()[0])
|
||||
if failed_count > 0:
|
||||
failed_hosts += 1
|
||||
else:
|
||||
successful_hosts += 1
|
||||
|
||||
if 'unreachable=' in line:
|
||||
unreachable = int(line.split('unreachable=')[1].split()[0])
|
||||
if unreachable > 0:
|
||||
unreachable_hosts += 1
|
||||
|
||||
if 'skipped=' in line:
|
||||
skipped = int(line.split('skipped=')[1].split()[0])
|
||||
if skipped > 0:
|
||||
skipped_hosts += 1
|
||||
|
||||
if 'changed=' in line:
|
||||
changed = int(line.split('changed=')[1].split()[0])
|
||||
if changed > 0:
|
||||
changed_hosts += 1
|
||||
|
||||
except (ValueError, IndexError):
|
||||
# Skip malformed lines
|
||||
continue
|
||||
|
||||
# Update execution record
|
||||
execution.successful_hosts = successful_hosts
|
||||
execution.failed_hosts = failed_hosts
|
||||
|
||||
# Match PLAY RECAP lines:
|
||||
# "RPI-FOO : ok=4 changed=1 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0"
|
||||
recap_re = re.compile(
|
||||
r'ok=(\d+)\s+changed=(\d+)\s+unreachable=(\d+)\s+failed=(\d+)'
|
||||
)
|
||||
|
||||
for line in output.split('\n'):
|
||||
m = recap_re.search(line)
|
||||
if not m:
|
||||
continue
|
||||
ok = int(m.group(1))
|
||||
changed = int(m.group(2))
|
||||
unreachable = int(m.group(3))
|
||||
failed = int(m.group(4))
|
||||
|
||||
if unreachable > 0:
|
||||
unreachable_hosts += 1
|
||||
elif failed > 0:
|
||||
failed_hosts += 1
|
||||
else:
|
||||
successful_hosts += 1
|
||||
|
||||
if changed > 0:
|
||||
changed_hosts += 1
|
||||
|
||||
execution.successful_hosts = successful_hosts
|
||||
execution.failed_hosts = failed_hosts
|
||||
execution.unreachable_hosts = unreachable_hosts
|
||||
execution.skipped_hosts = skipped_hosts
|
||||
execution.changed_hosts = changed_hosts
|
||||
execution.skipped_hosts = skipped_hosts
|
||||
execution.changed_hosts = changed_hosts
|
||||
|
||||
def _get_playbook_description(self, playbook_name: str) -> str:
|
||||
"""Get user-friendly description for playbook"""
|
||||
descriptions = {
|
||||
'update_devices': 'Update all packages and monitoring software on devices',
|
||||
'restart_service': 'Restart monitoring services on selected devices',
|
||||
'restart_service': 'Restart monitoring services on selected devices',
|
||||
'system_health': 'Check system health and monitoring status',
|
||||
'maintenance_mode': 'Put devices in maintenance mode'
|
||||
'maintenance_mode': 'Put devices in maintenance mode',
|
||||
'distribute_ssh_keys': 'Push server public key to all devices using password auth',
|
||||
}
|
||||
return descriptions.get(playbook_name, f'Execute {playbook_name} playbook')
|
||||
|
||||
def create_distribute_ssh_keys_playbook(self) -> str:
|
||||
"""Ensure the distribute_ssh_keys playbook file exists (ships with the repo)."""
|
||||
playbook_path = self.playbook_dir / 'distribute_ssh_keys.yml'
|
||||
if not playbook_path.exists():
|
||||
logging.warning('distribute_ssh_keys.yml not found — playbook file is missing')
|
||||
return str(playbook_path)
|
||||
|
||||
def create_system_health_playbook(self) -> str:
|
||||
"""Create system health check playbook"""
|
||||
@@ -782,6 +819,53 @@ class AnsibleService:
|
||||
unreachable = int(line.split('unreachable=')[1].split()[0])
|
||||
execution.unreachable_hosts += unreachable
|
||||
|
||||
def test_password_auth(self, device_ip: str, password: str,
|
||||
username: str = 'pi', port: int = 22) -> Dict:
|
||||
"""
|
||||
Test SSH connectivity using password-only authentication (no key fallback).
|
||||
Uses sshpass so we can confirm the exact password works before deploying keys.
|
||||
"""
|
||||
try:
|
||||
# Quick TCP reachability check first
|
||||
import socket
|
||||
with socket.create_connection((device_ip, port), timeout=5):
|
||||
pass
|
||||
except (OSError, ConnectionRefusedError) as e:
|
||||
return {'success': False, 'reachable': False,
|
||||
'error': f'Host unreachable on port {port}: {e}'}
|
||||
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
'sshpass', '-p', password,
|
||||
'ssh',
|
||||
'-o', 'PubkeyAuthentication=no',
|
||||
'-o', 'PreferredAuthentications=password',
|
||||
'-o', 'StrictHostKeyChecking=no',
|
||||
'-o', 'UserKnownHostsFile=/dev/null',
|
||||
'-o', f'ConnectTimeout=8',
|
||||
'-p', str(port),
|
||||
f'{username}@{device_ip}',
|
||||
'echo OK',
|
||||
],
|
||||
capture_output=True, text=True, timeout=15,
|
||||
)
|
||||
if result.returncode == 0 and 'OK' in result.stdout:
|
||||
return {'success': True, 'reachable': True,
|
||||
'message': f'Password authentication succeeded for {username}@{device_ip}'}
|
||||
else:
|
||||
stderr = (result.stderr or '').strip()
|
||||
return {'success': False, 'reachable': True,
|
||||
'error': f'Authentication failed — {stderr or "wrong password"}'}
|
||||
except subprocess.TimeoutExpired:
|
||||
return {'success': False, 'reachable': True,
|
||||
'error': 'SSH command timed out'}
|
||||
except FileNotFoundError:
|
||||
return {'success': False, 'reachable': True,
|
||||
'error': 'sshpass not installed — run: sudo apt-get install sshpass'}
|
||||
except Exception as e:
|
||||
return {'success': False, 'reachable': True, 'error': str(e)}
|
||||
|
||||
def test_ssh_connectivity(self, device_ip: str, username: str = 'pi') -> Dict:
|
||||
"""Test SSH connectivity to a device"""
|
||||
try:
|
||||
|
||||
@@ -3,7 +3,7 @@ Web routes for Ansible management interface
|
||||
"""
|
||||
from flask import Blueprint, render_template, request, redirect, url_for, flash, jsonify
|
||||
from app.services.ansible_service import AnsibleService
|
||||
from app.models import Device, AnsibleExecution, PlaybookExecution
|
||||
from app.models import Device, AnsibleExecution, PlaybookExecution, ExecutionFailureReport
|
||||
from config.database_config import get_db
|
||||
import logging
|
||||
|
||||
@@ -89,13 +89,18 @@ def playbooks():
|
||||
'builtin': True
|
||||
},
|
||||
{
|
||||
'name': 'restart_service',
|
||||
'name': 'restart_service',
|
||||
'description': 'Restart monitoring services on devices',
|
||||
'builtin': True
|
||||
}
|
||||
},
|
||||
{
|
||||
'name': 'distribute_ssh_keys',
|
||||
'description': 'Push server public key to devices using password auth',
|
||||
'builtin': True
|
||||
},
|
||||
]
|
||||
|
||||
return render_template('ansible/playbooks.html',
|
||||
return render_template('ansible/playbooks.html',
|
||||
playbooks=playbooks,
|
||||
builtin_playbooks=builtin_playbooks)
|
||||
except Exception as e:
|
||||
@@ -123,17 +128,20 @@ def execute():
|
||||
})
|
||||
seen.add(h['hostname'])
|
||||
|
||||
settings = ansible_service.load_settings()
|
||||
return render_template('ansible/execute.html',
|
||||
inventory=inventory_data,
|
||||
all_inv_hosts=all_inv_hosts,
|
||||
preselect_playbook=preselect)
|
||||
preselect_playbook=preselect,
|
||||
use_password_auth=settings.get('use_password_auth', False))
|
||||
except Exception as e:
|
||||
logging.error(f"Error loading execute form: {e}")
|
||||
flash(f'Error loading form: {e}', 'error')
|
||||
return render_template('ansible/execute.html',
|
||||
inventory={'groups': {}},
|
||||
all_inv_hosts=[],
|
||||
preselect_playbook='')
|
||||
preselect_playbook='',
|
||||
use_password_auth=False)
|
||||
|
||||
elif request.method == 'POST':
|
||||
# Execute playbook
|
||||
@@ -182,17 +190,24 @@ def execute():
|
||||
ansible_service.create_restart_service_playbook()
|
||||
elif playbook_name == 'system_health':
|
||||
ansible_service.create_system_health_playbook()
|
||||
|
||||
elif playbook_name == 'distribute_ssh_keys':
|
||||
ansible_service.create_distribute_ssh_keys_playbook()
|
||||
|
||||
# Add controller IP for callbacks
|
||||
extra_vars['ansible_controller_ip'] = request.host
|
||||
|
||||
# Force password auth for key distribution, or honour the form toggle
|
||||
force_password = (playbook_name == 'distribute_ssh_keys') or \
|
||||
bool(request.form.get('force_password_auth'))
|
||||
|
||||
# Use async execution (returns immediately with execution_id)
|
||||
result = ansible_service.execute_playbook_async(
|
||||
playbook_name=playbook_name,
|
||||
limit_hosts=selected_hosts,
|
||||
extra_vars=extra_vars,
|
||||
priority=priority,
|
||||
max_retries=max_retries
|
||||
max_retries=max_retries,
|
||||
force_password_auth=force_password,
|
||||
)
|
||||
|
||||
if result['success']:
|
||||
@@ -256,6 +271,11 @@ def execution_details(execution_id):
|
||||
flash(f'Error loading execution details: {e}', 'error')
|
||||
return redirect(url_for('ansible_web.executions'))
|
||||
|
||||
@ansible_web_bp.route('/executions/<execution_id>/live-popup')
|
||||
def execution_live_popup(execution_id):
|
||||
"""Standalone popup window for live execution output"""
|
||||
return render_template('ansible/live_popup.html', execution_id=execution_id)
|
||||
|
||||
@ansible_web_bp.route('/ssh/setup')
|
||||
def ssh_setup():
|
||||
"""SSH key setup interface"""
|
||||
@@ -288,10 +308,14 @@ def save_ssh_settings():
|
||||
try:
|
||||
fallback_password = request.form.get('ssh_fallback_password', '').strip()
|
||||
if not fallback_password:
|
||||
flash('Fallback password cannot be empty.', 'error')
|
||||
flash('Password cannot be empty.', 'error')
|
||||
return redirect(url_for('ansible_web.ssh_setup'))
|
||||
|
||||
ansible_service.save_settings({'ssh_fallback_password': fallback_password})
|
||||
use_password_auth = request.form.get('use_password_auth') == 'on'
|
||||
ansible_service.save_settings({
|
||||
'ssh_fallback_password': fallback_password,
|
||||
'use_password_auth': use_password_auth,
|
||||
})
|
||||
flash('SSH settings saved successfully.', 'success')
|
||||
except Exception as e:
|
||||
logging.error(f"Error saving SSH settings: {e}")
|
||||
@@ -596,4 +620,20 @@ def delete_playbook():
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error deleting playbook: {e}")
|
||||
return jsonify({'error': str(e)}), 500
|
||||
return jsonify({'error': str(e)}), 500
|
||||
|
||||
|
||||
@ansible_web_bp.route('/failure-reports')
|
||||
def failure_reports():
|
||||
"""View all saved execution failure reports."""
|
||||
try:
|
||||
with get_db().get_session() as session:
|
||||
reports = session.query(ExecutionFailureReport)\
|
||||
.order_by(ExecutionFailureReport.saved_at.desc())\
|
||||
.all()
|
||||
reports_data = [r.to_dict() for r in reports]
|
||||
return render_template('ansible/failure_reports.html', reports=reports_data)
|
||||
except Exception as e:
|
||||
logging.error(f"Error loading failure reports: {e}")
|
||||
flash(f'Error loading failure reports: {e}', 'error')
|
||||
return render_template('ansible/failure_reports.html', reports=[])
|
||||
Reference in New Issue
Block a user