ske087/Server_Monitorizare_v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updated view and database

Browse Source
This commit is contained in:
ske087
2026-06-24 15:21:40 +03:00
parent cb12a8f1cf
commit a48b3afb83
14 changed files with 372 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/wmt.py
+21 -2
View File
@@ -42,6 +42,20 @@ def _latest_config_ts(session, mac_address):
return global_ts, device_ts, latest
def _mark_wmt_checkin(device):
"""Stamp a device as having checked in via the WMT client API.
Keeps the unified device record in sync: both the general monitoring
'last_seen' and the WMT-specific 'wmt_last_seen' are advanced on every
client interaction so the unified devices page reflects live state.
"""
if device is None:
return
now = datetime.utcnow()
device.last_seen = now
device.wmt_last_seen = now
# ---------------------------------------------------------------------------
# Endpoints
# ---------------------------------------------------------------------------
@@ -67,6 +81,10 @@ def get_config_timestamp():
with get_db().get_session() as session:
global_ts, device_info_reviewed_ts, latest = _latest_config_ts(session, mac)
# Stamp WMT check-in so the unified device record stays live
device = session.query(Device).filter_by(mac_address=mac).first()
_mark_wmt_checkin(device)
return jsonify({
'global_updated_at': global_ts.isoformat() if global_ts != datetime(1970, 1, 1) else None,
'device_info_reviewed_at': device_info_reviewed_ts.isoformat() if device_info_reviewed_ts != datetime(1970, 1, 1) else None,
@@ -91,9 +109,9 @@ def get_device_config(mac_address):
global_cfg = _get_or_create_global_config(session)
device = session.query(Device).filter_by(mac_address=mac).first()
# Update last_seen if device is known
# Update last_seen / wmt_last_seen if device is known
if device:
device.last_seen = datetime.utcnow()
_mark_wmt_checkin(device)
_, device_ts, latest_ts = _latest_config_ts(session, mac)
@@ -214,6 +232,7 @@ def submit_update_request():
# Device is known from here on ─────────────────────────────
device.last_seen = datetime.utcnow()
_mark_wmt_checkin(device)
if card_presence in ('enable', 'disable'):
device.card_presence = card_presence
app/models/__init__.py
+27 -1
View File
@@ -4,7 +4,7 @@ Database models for enhanced server monitoring system
from sqlalchemy import Column, Integer, String, DateTime, Text, Boolean, ForeignKey, LargeBinary, Float, Table
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from datetime import datetime
from datetime import datetime, timedelta
import json
import hashlib
from cryptography.fernet import Fernet
@@ -53,6 +53,9 @@ class Device(Base):
info_reviewed_at = Column(DateTime, default=lambda: datetime(1970, 1, 1))
card_presence = Column(String(10), default='enable')
custom_chrome_url = Column(String(500), nullable=True) # per-device production URL override (overrides WMTGlobalConfig.chrome_url)
# Stamped on every WMT client check-in (config pull / update request / timestamp poll).
# Authoritative signal that a device is "WMT enabled" on the unified devices page.
wmt_last_seen = Column(DateTime, nullable=True, index=True)
# Relationships
logs = relationship("LogEntry", back_populates="device")
@@ -67,6 +70,29 @@ class Device(Base):
"""Alias for nume_masa – used by WMT module."""
return self.nume_masa
@property
def wmt_enabled(self):
"""True when this device has ever checked in via the WMT client API.
This is the authoritative "WMT enabled" signal for the unified devices
page (replaces the old 'has a MAC address' heuristic).
"""
return self.wmt_last_seen is not None
@property
def is_online(self):
"""Heuristic online state: active status and seen within the last 10 minutes."""
if self.status != 'active' or not self.last_seen:
return False
return (datetime.utcnow() - self.last_seen) <= timedelta(minutes=10)
@property
def minutes_since_last_seen(self):
"""Whole minutes since the device was last seen (None if never seen)."""
if not self.last_seen:
return None
return int((datetime.utcnow() - self.last_seen).total_seconds() // 60)
def __repr__(self):
return f"<Device(hostname='{self.hostname}', ip='{self.device_ip}')>"
app/services/ansible_service.py
+8 -1
View File
@@ -27,7 +27,7 @@ class AnsibleService:
}
def __init__(self):
self.db = get_db()
self._db = None
self.ansible_dir = Path("ansible")
self.inventory_file = self.ansible_dir / "inventory" / "dynamic_inventory.yaml"
self.playbook_dir = self.ansible_dir / "playbooks"
@@ -42,6 +42,13 @@ class AnsibleService:
(self.ansible_dir / "roles").mkdir(exist_ok=True)
self.ssh_keys_dir.mkdir(mode=0o700, exist_ok=True)
@property
def db(self):
"""Lazily resolve the database handle (see LogCompressionService.db)."""
if self._db is None:
self._db = get_db()
return self._db
# ------------------------------------------------------------------ #
# Settings helpers #
# ------------------------------------------------------------------ #
app/services/device_service.py
+12 -2
View File
@@ -13,8 +13,15 @@ class DeviceService:
"""Service for managing devices and device-related operations"""
def __init__(self):
self.db = get_db()
self._db = None
self.logger = logging.getLogger(__name__)
@property
def db(self):
"""Lazily resolve the database handle (see LogCompressionService.db)."""
if self._db is None:
self._db = get_db()
return self._db
# Basic CRUD Operations
@@ -213,7 +220,10 @@ class DeviceService:
'recent_logs_24h': recent_logs,
'total_files': total_files,
'last_log': last_log,
'uptime_days': (datetime.utcnow() - device.last_seen).days if device.last_seen else 0
# Days since the device was last seen (0 = seen today / currently active).
# Previously mislabeled as "uptime_days" which implied the opposite.
'days_since_last_seen': (datetime.utcnow() - device.last_seen).days if device.last_seen else None,
'is_online': device.is_online,
}
except Exception as e:
app/services/file_service.py
+33 -26
View File
@@ -9,13 +9,14 @@ from pathlib import Path
from werkzeug.utils import secure_filename
from app.models import Device, FileUpload
from config.database_config import get_db
from sqlalchemy import func
import logging
class FileUploadService:
"""Service for handling file uploads and processing"""
def __init__(self):
self.db = get_db()
self._db = None
self.upload_folder = Path("data/uploads")
self.upload_folder.mkdir(exist_ok=True)
@@ -27,7 +28,14 @@ class FileUploadService:
# Max file size (50MB)
self.max_file_size = 50 * 1024 * 1024
@property
def db(self):
"""Lazily resolve the database handle (see LogCompressionService.db)."""
if self._db is None:
self._db = get_db()
return self._db
def process_uploaded_file(self, file, device_info):
"""Process uploaded file from device"""
try:
@@ -105,7 +113,7 @@ class FileUploadService:
# Process file content if it's a log file
if self._is_log_file(filename, mime_type):
self._process_log_file(file_upload, file_content)
self._process_log_file(session, file_upload, file_content)
return {
'success': True,
@@ -176,51 +184,50 @@ class FileUploadService:
return False
def _process_log_file(self, file_upload, content):
"""Process log file content to extract log entries"""
def _process_log_file(self, session, file_upload, content):
"""Process log file content to extract log entries.
Reuses the caller's database session (no nested per-line sessions) by
calling LogCompressionService.store_log_in_session for every line.
"""
try:
# Mark as log file
file_upload.is_log_file = True
# Simple log processing - split by lines
lines = content.decode('utf-8', errors='ignore').split('\n')
entries_extracted = 0
from app.services.log_service import LogCompressionService
log_service = LogCompressionService()
device_info = {
'hostname': file_upload.device.hostname,
'device_ip': file_upload.device.device_ip,
'nume_masa': file_upload.device.nume_masa
}
device = file_upload.device
for line_num, line in enumerate(lines, 1):
line = line.strip()
if not line:
continue
# Try to extract timestamp and message
# This is a simple implementation - enhance as needed
message = f"[File: {file_upload.original_filename}:{line_num}] {line}"
# Process through log compression service
result = log_service.process_log_message(
device_info=device_info,
# Store within the SAME session – avoids opening a session per line
log_service.store_log_in_session(
session=session,
device=device,
message=message,
severity='info'
)
if result['success']:
entries_extracted += 1
entries_extracted += 1
# Update file record
file_upload.log_entries_extracted = entries_extracted
file_upload.processed = True
file_upload.processing_status = 'completed'
logging.info(f"Processed log file {file_upload.filename}: {entries_extracted} entries extracted")
except Exception as e:
logging.error(f"Error processing log file content: {e}")
file_upload.processing_status = 'error'
@@ -235,7 +242,7 @@ class FileUploadService:
# Calculate total size
total_size = session.query(
session.func.sum(FileUpload.file_size)
func.sum(FileUpload.file_size)
).scalar() or 0
# Count by processing status
app/services/log_service.py
+134 -112
View File
@@ -15,47 +15,66 @@ class LogCompressionService:
"""Service for compressing log messages using templates and aliases"""
def __init__(self):
self.db = get_db()
self._db = None
self.template_patterns = self._load_common_patterns()
@property
def db(self):
"""Lazily resolve the database handle so creating this service at import
time does not initialize the DB before the Flask app context exists."""
if self._db is None:
self._db = get_db()
return self._db
def _load_common_patterns(self) -> List[Dict]:
"""Load common log message patterns for template matching"""
"""Load common log message patterns for template matching.
Each entry's 'variables' list maps regex capture groups (in order) to
template variable names, so matching/extraction is fully data-driven and
no longer relies on a hardcoded if/elif chain.
"""
return [
{
'pattern': r'Card detected: ([A-F0-9]+)',
'template': 'Card detected: {card_id}',
'category': 'card_detection',
'alias_prefix': 'CD'
'alias_prefix': 'CD',
'variables': ['card_id'],
},
{
'pattern': r'Connection failed: (.+)',
'template': 'Connection failed: {error}',
'category': 'connection_error',
'alias_prefix': 'CE'
'alias_prefix': 'CE',
'variables': ['error'],
},
{
'pattern': r'System startup completed in ([0-9.]+)s',
'template': 'System startup completed in {time}s',
'category': 'system_startup',
'alias_prefix': 'SS'
'alias_prefix': 'SS',
'variables': ['time'],
},
{
'pattern': r'Auto-update: (.+)',
'template': 'Auto-update: {message}',
'category': 'auto_update',
'alias_prefix': 'AU'
'alias_prefix': 'AU',
'variables': ['message'],
},
{
'pattern': r'Command \'([^\']+)\' (SUCCESS|FAILED)',
'template': 'Command \'{command}\' {status}',
'category': 'command_execution',
'alias_prefix': 'EX'
'alias_prefix': 'EX',
'variables': ['command', 'status'],
},
{
'pattern': r'Temperature: ([0-9.]+)°C',
'template': 'Temperature: {temp}°C',
'category': 'temperature',
'alias_prefix': 'TM'
'alias_prefix': 'TM',
'variables': ['temp'],
}
]
@@ -75,75 +94,12 @@ class LogCompressionService:
with self.db.get_session() as session:
# Get or create device
device = self._get_or_create_device(session, device_info)
# Try to match message to existing template
template, variables = self._match_message_template(session, message)
if template:
# Use existing template
log_entry = LogEntry(
device_id=device.id,
template_id=template.id,
template_variables=json.dumps(variables) if variables else None,
severity=severity,
timestamp=datetime.utcnow()
)
# Update template usage count
template.usage_count += 1
# Calculate size savings
original_size = len(message.encode('utf-8'))
compressed_size = len(template.alias.encode('utf-8')) + \
len(json.dumps(variables or {}).encode('utf-8'))
compression_info = {
'used_template': True,
'template_alias': template.alias,
'original_size': original_size,
'compressed_size': compressed_size,
'savings_percent': ((original_size - compressed_size) / original_size) * 100
}
else:
# Create new template if message matches a pattern
template = self._create_new_template(session, message)
if template:
# New template created
variables = self._extract_variables(message, template.template_text)
log_entry = LogEntry(
device_id=device.id,
template_id=template.id,
template_variables=json.dumps(variables) if variables else None,
severity=severity,
timestamp=datetime.utcnow()
)
template.usage_count = 1
compression_info = {
'used_template': True,
'template_alias': template.alias,
'new_template': True,
'original_size': len(message.encode('utf-8')),
'compressed_size': len(template.alias.encode('utf-8'))
}
else:
# Store as full message
log_entry = LogEntry(
device_id=device.id,
full_message=message,
severity=severity,
timestamp=datetime.utcnow()
)
compression_info = {
'used_template': False,
'stored_full': True,
'original_size': len(message.encode('utf-8'))
}
session.add(log_entry)
log_entry, compression_info = self.store_log_in_session(
session, device, message, severity
)
session.flush() # Get the log entry ID
return {
'success': True,
'log_id': log_entry.id,
@@ -159,7 +115,73 @@ class LogCompressionService:
'error': str(e),
'message': 'Log processing failed'
}
def store_log_in_session(self, session: Session, device: Device,
message: str, severity: str = 'info') -> Tuple[LogEntry, Dict]:
"""Compress and store a single log message using an existing session/device.
Shared by process_log_message() and bulk file ingestion so callers don't
open a new database session per log line. The caller is responsible for
committing (the session context manager does this).
Returns (log_entry, compression_info).
"""
# Try to match message to existing template
template, variables = self._match_message_template(session, message)
if template:
log_entry = LogEntry(
device_id=device.id,
template_id=template.id,
template_variables=json.dumps(variables) if variables else None,
severity=severity,
timestamp=datetime.utcnow()
)
template.usage_count += 1
original_size = len(message.encode('utf-8'))
compressed_size = len(template.alias.encode('utf-8')) + \
len(json.dumps(variables or {}).encode('utf-8'))
compression_info = {
'used_template': True,
'template_alias': template.alias,
'original_size': original_size,
'compressed_size': compressed_size,
'savings_percent': ((original_size - compressed_size) / original_size) * 100
}
else:
template, variables = self._create_new_template(session, message)
if template:
log_entry = LogEntry(
device_id=device.id,
template_id=template.id,
template_variables=json.dumps(variables) if variables else None,
severity=severity,
timestamp=datetime.utcnow()
)
template.usage_count = 1
compression_info = {
'used_template': True,
'template_alias': template.alias,
'new_template': True,
'original_size': len(message.encode('utf-8')),
'compressed_size': len(template.alias.encode('utf-8'))
}
else:
log_entry = LogEntry(
device_id=device.id,
full_message=message,
severity=severity,
timestamp=datetime.utcnow()
)
compression_info = {
'used_template': False,
'stored_full': True,
'original_size': len(message.encode('utf-8'))
}
session.add(log_entry)
return log_entry, compression_info
@staticmethod
def _infer_device_type(hostname: str) -> str:
"""Guess device type from hostname pattern."""
@@ -250,49 +272,37 @@ class LogCompressionService:
template_text=template_text,
category=pattern_info['category']
).first()
if template:
# Extract variables
variables = {}
for i, group in enumerate(match.groups(), 1):
# Map to variable names based on template
if '{card_id}' in template_text and pattern_info['category'] == 'card_detection':
variables['card_id'] = group
elif '{error}' in template_text and pattern_info['category'] == 'connection_error':
variables['error'] = group
elif '{time}' in template_text and pattern_info['category'] == 'system_startup':
variables['time'] = group
elif '{message}' in template_text:
variables['message'] = group
elif '{command}' in template_text and i == 1:
variables['command'] = group
elif '{status}' in template_text and i == 2:
variables['status'] = group
elif '{temp}' in template_text:
variables['temp'] = group
variables = self._variables_from_match(pattern_info, match)
return template, variables
return None, None
def _create_new_template(self, session: Session, message: str) -> Optional[MessageTemplate]:
"""Create new template if message matches a known pattern"""
def _create_new_template(self, session: Session, message: str) -> Tuple[Optional[MessageTemplate], Optional[Dict]]:
"""Create new template if message matches a known pattern.
Returns (template, variables) so the caller can persist the extracted
variables on the first log entry that creates the template.
"""
for pattern_info in self.template_patterns:
match = re.match(pattern_info['pattern'], message)
if match:
variables = self._variables_from_match(pattern_info, match)
# Check if template already exists
existing = session.query(MessageTemplate).filter_by(
template_text=pattern_info['template'],
category=pattern_info['category']
).first()
if existing:
return existing
return existing, variables
# Create new template
alias = self._generate_alias(session, pattern_info['alias_prefix'])
template_hash = MessageTemplate.create_hash(pattern_info['template'])
template = MessageTemplate(
template_hash=template_hash,
template_text=pattern_info['template'],
@@ -303,9 +313,9 @@ class LogCompressionService:
session.add(template)
session.flush()
return template
return template, variables
return None
return None, None
def _generate_alias(self, session: Session, prefix: str) -> str:
"""Generate unique alias for template"""
@@ -325,12 +335,24 @@ class LogCompressionService:
return f"{prefix}{max_num + 1:03d}"
def _extract_variables(self, message: str, template: str) -> Dict:
"""Extract variables from message using template"""
# Simple variable extraction - could be enhanced
variables = {}
# This is a simplified implementation
# In production, you'd want more sophisticated template matching
return variables
"""Extract template variables from a raw message.
Finds the matching pattern for the message and maps its capture groups
to the configured variable names. Returns {} when no pattern matches.
"""
for pattern_info in self.template_patterns:
if pattern_info['template'] != template:
continue
match = re.match(pattern_info['pattern'], message)
if match:
return self._variables_from_match(pattern_info, match)
return {}
@staticmethod
def _variables_from_match(pattern_info: Dict, match) -> Dict:
"""Map a regex match's capture groups to named template variables."""
names = pattern_info.get('variables') or []
return {name: value for name, value in zip(names, match.groups())}
def get_compression_stats(self) -> Dict:
"""Get compression statistics"""
app/web/main.py
+22 -18
View File
@@ -2,7 +2,7 @@
Main web routes for dashboard and device management
"""
from flask import Blueprint, render_template, request, redirect, url_for, flash, jsonify
from app.models import Device, LogEntry, MessageTemplate, AnsibleExecution, WMTUpdateRequest, InventoryGroup, device_inventory_association
from app.models import Device, LogEntry, MessageTemplate, AnsibleExecution, PlaybookExecution, WMTUpdateRequest, InventoryGroup, device_inventory_association
from config.database_config import get_db
from app.services.log_service import LogCompressionService
from datetime import datetime, timedelta
@@ -242,10 +242,12 @@ def stats():
# Get execution statistics
exec_stats = {
'total': session.query(AnsibleExecution).count(),
'successful': session.query(AnsibleExecution).filter_by(status='completed').count(),
'failed': session.query(AnsibleExecution).filter_by(status='failed').count(),
'running': session.query(AnsibleExecution).filter_by(status='running').count()
'total': session.query(PlaybookExecution).count(),
'successful': session.query(PlaybookExecution).filter_by(status='completed').count(),
'failed': session.query(PlaybookExecution).filter_by(status='failed').count(),
'running': session.query(PlaybookExecution).filter(
PlaybookExecution.status.in_(['queued', 'running'])
).count()
}
return render_template('stats.html',
@@ -541,6 +543,21 @@ def admin():
@main_bp.route('/admin/clear/logs', methods=['POST'])
def admin_clear_logs():
"""Delete all log entries from the database."""
return _clear_all_log_entries()
@main_bp.route('/admin/clear/device-logs', methods=['POST'])
def admin_clear_device_logs():
"""Delete all log entries from the database (devices stay intact).
Functionally identical to admin_clear_logs – both endpoints are kept because
the admin UI exposes two separate buttons, but they share one implementation.
"""
return _clear_all_log_entries()
def _clear_all_log_entries():
"""Shared implementation: delete every LogEntry row."""
try:
with get_db().get_session() as session:
count = session.query(LogEntry).delete()
@@ -551,19 +568,6 @@ def admin_clear_logs():
return jsonify({'success': False, 'error': str(e)}), 500
@main_bp.route('/admin/clear/device-logs', methods=['POST'])
def admin_clear_device_logs():
"""Delete all log entries from the database (devices stay intact)."""
try:
with get_db().get_session() as session:
count = session.query(LogEntry).delete()
session.commit()
return jsonify({'success': True, 'deleted': count})
except Exception as e:
logging.error(f'Admin clear device logs error: {e}')
return jsonify({'success': False, 'error': str(e)}), 500
@main_bp.route('/admin/delete/device/<int:device_id>', methods=['POST'])
def admin_delete_device(device_id):
"""Delete a single registered device and its log entries."""
app/web/wmt.py
+2 -21
View File
@@ -216,27 +216,8 @@ def reject_request(req_id):
@wmt_web_bp.route('/devices')
def devices():
"""List all WMT-registered devices."""
try:
with get_db().get_session() as session:
device_list = (
session.query(Device)
.filter(Device.mac_address.isnot(None))
.order_by(Device.nume_masa)
.all()
)
return render_template(
'wmt/devices.html',
devices=device_list,
breadcrumbs=[
{'url': url_for('wmt_web.index'), 'title': 'WMT Management'},
{'url': url_for('wmt_web.devices'), 'title': 'Devices'},
],
)
except Exception as e:
logger.error(f'WMT devices list error: {e}')
flash(f'Error: {e}', 'error')
return redirect(url_for('wmt_web.index'))
"""Retired: WMT devices are now shown on the unified Devices page."""
return redirect(url_for('main.devices'))
@wmt_web_bp.route('/devices/new', methods=['GET', 'POST'])