ske087/Server_Monitorizare_v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updated view and database

Browse Source
This commit is contained in:
ske087
2026-06-24 15:21:40 +03:00
parent cb12a8f1cf
commit a48b3afb83
14 changed files with 372 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/ansible_service.py
+8 -1
View File
@@ -27,7 +27,7 @@ class AnsibleService:
}
def __init__(self):
self.db = get_db()
self._db = None
self.ansible_dir = Path("ansible")
self.inventory_file = self.ansible_dir / "inventory" / "dynamic_inventory.yaml"
self.playbook_dir = self.ansible_dir / "playbooks"
@@ -42,6 +42,13 @@ class AnsibleService:
(self.ansible_dir / "roles").mkdir(exist_ok=True)
self.ssh_keys_dir.mkdir(mode=0o700, exist_ok=True)
@property
def db(self):
"""Lazily resolve the database handle (see LogCompressionService.db)."""
if self._db is None:
self._db = get_db()
return self._db
# ------------------------------------------------------------------ #
# Settings helpers #
# ------------------------------------------------------------------ #
app/services/device_service.py
+12 -2
View File
@@ -13,8 +13,15 @@ class DeviceService:
"""Service for managing devices and device-related operations"""
def __init__(self):
self.db = get_db()
self._db = None
self.logger = logging.getLogger(__name__)
@property
def db(self):
"""Lazily resolve the database handle (see LogCompressionService.db)."""
if self._db is None:
self._db = get_db()
return self._db
# Basic CRUD Operations
@@ -213,7 +220,10 @@ class DeviceService:
'recent_logs_24h': recent_logs,
'total_files': total_files,
'last_log': last_log,
'uptime_days': (datetime.utcnow() - device.last_seen).days if device.last_seen else 0
# Days since the device was last seen (0 = seen today / currently active).
# Previously mislabeled as "uptime_days" which implied the opposite.
'days_since_last_seen': (datetime.utcnow() - device.last_seen).days if device.last_seen else None,
'is_online': device.is_online,
}
except Exception as e:
app/services/file_service.py
+33 -26
View File
@@ -9,13 +9,14 @@ from pathlib import Path
from werkzeug.utils import secure_filename
from app.models import Device, FileUpload
from config.database_config import get_db
from sqlalchemy import func
import logging
class FileUploadService:
"""Service for handling file uploads and processing"""
def __init__(self):
self.db = get_db()
self._db = None
self.upload_folder = Path("data/uploads")
self.upload_folder.mkdir(exist_ok=True)
@@ -27,7 +28,14 @@ class FileUploadService:
# Max file size (50MB)
self.max_file_size = 50 * 1024 * 1024
@property
def db(self):
"""Lazily resolve the database handle (see LogCompressionService.db)."""
if self._db is None:
self._db = get_db()
return self._db
def process_uploaded_file(self, file, device_info):
"""Process uploaded file from device"""
try:
@@ -105,7 +113,7 @@ class FileUploadService:
# Process file content if it's a log file
if self._is_log_file(filename, mime_type):
self._process_log_file(file_upload, file_content)
self._process_log_file(session, file_upload, file_content)
return {
'success': True,
@@ -176,51 +184,50 @@ class FileUploadService:
return False
def _process_log_file(self, file_upload, content):
"""Process log file content to extract log entries"""
def _process_log_file(self, session, file_upload, content):
"""Process log file content to extract log entries.
Reuses the caller's database session (no nested per-line sessions) by
calling LogCompressionService.store_log_in_session for every line.
"""
try:
# Mark as log file
file_upload.is_log_file = True
# Simple log processing - split by lines
lines = content.decode('utf-8', errors='ignore').split('\n')
entries_extracted = 0
from app.services.log_service import LogCompressionService
log_service = LogCompressionService()
device_info = {
'hostname': file_upload.device.hostname,
'device_ip': file_upload.device.device_ip,
'nume_masa': file_upload.device.nume_masa
}
device = file_upload.device
for line_num, line in enumerate(lines, 1):
line = line.strip()
if not line:
continue
# Try to extract timestamp and message
# This is a simple implementation - enhance as needed
message = f"[File: {file_upload.original_filename}:{line_num}] {line}"
# Process through log compression service
result = log_service.process_log_message(
device_info=device_info,
# Store within the SAME session – avoids opening a session per line
log_service.store_log_in_session(
session=session,
device=device,
message=message,
severity='info'
)
if result['success']:
entries_extracted += 1
entries_extracted += 1
# Update file record
file_upload.log_entries_extracted = entries_extracted
file_upload.processed = True
file_upload.processing_status = 'completed'
logging.info(f"Processed log file {file_upload.filename}: {entries_extracted} entries extracted")
except Exception as e:
logging.error(f"Error processing log file content: {e}")
file_upload.processing_status = 'error'
@@ -235,7 +242,7 @@ class FileUploadService:
# Calculate total size
total_size = session.query(
session.func.sum(FileUpload.file_size)
func.sum(FileUpload.file_size)
).scalar() or 0
# Count by processing status
app/services/log_service.py
+134 -112
View File
@@ -15,47 +15,66 @@ class LogCompressionService:
"""Service for compressing log messages using templates and aliases"""
def __init__(self):
self.db = get_db()
self._db = None
self.template_patterns = self._load_common_patterns()
@property
def db(self):
"""Lazily resolve the database handle so creating this service at import
time does not initialize the DB before the Flask app context exists."""
if self._db is None:
self._db = get_db()
return self._db
def _load_common_patterns(self) -> List[Dict]:
"""Load common log message patterns for template matching"""
"""Load common log message patterns for template matching.
Each entry's 'variables' list maps regex capture groups (in order) to
template variable names, so matching/extraction is fully data-driven and
no longer relies on a hardcoded if/elif chain.
"""
return [
{
'pattern': r'Card detected: ([A-F0-9]+)',
'template': 'Card detected: {card_id}',
'category': 'card_detection',
'alias_prefix': 'CD'
'alias_prefix': 'CD',
'variables': ['card_id'],
},
{
'pattern': r'Connection failed: (.+)',
'template': 'Connection failed: {error}',
'category': 'connection_error',
'alias_prefix': 'CE'
'alias_prefix': 'CE',
'variables': ['error'],
},
{
'pattern': r'System startup completed in ([0-9.]+)s',
'template': 'System startup completed in {time}s',
'category': 'system_startup',
'alias_prefix': 'SS'
'alias_prefix': 'SS',
'variables': ['time'],
},
{
'pattern': r'Auto-update: (.+)',
'template': 'Auto-update: {message}',
'category': 'auto_update',
'alias_prefix': 'AU'
'alias_prefix': 'AU',
'variables': ['message'],
},
{
'pattern': r'Command \'([^\']+)\' (SUCCESS|FAILED)',
'template': 'Command \'{command}\' {status}',
'category': 'command_execution',
'alias_prefix': 'EX'
'alias_prefix': 'EX',
'variables': ['command', 'status'],
},
{
'pattern': r'Temperature: ([0-9.]+)°C',
'template': 'Temperature: {temp}°C',
'category': 'temperature',
'alias_prefix': 'TM'
'alias_prefix': 'TM',
'variables': ['temp'],
}
]
@@ -75,75 +94,12 @@ class LogCompressionService:
with self.db.get_session() as session:
# Get or create device
device = self._get_or_create_device(session, device_info)
# Try to match message to existing template
template, variables = self._match_message_template(session, message)
if template:
# Use existing template
log_entry = LogEntry(
device_id=device.id,
template_id=template.id,
template_variables=json.dumps(variables) if variables else None,
severity=severity,
timestamp=datetime.utcnow()
)
# Update template usage count
template.usage_count += 1
# Calculate size savings
original_size = len(message.encode('utf-8'))
compressed_size = len(template.alias.encode('utf-8')) + \
len(json.dumps(variables or {}).encode('utf-8'))
compression_info = {
'used_template': True,
'template_alias': template.alias,
'original_size': original_size,
'compressed_size': compressed_size,
'savings_percent': ((original_size - compressed_size) / original_size) * 100
}
else:
# Create new template if message matches a pattern
template = self._create_new_template(session, message)
if template:
# New template created
variables = self._extract_variables(message, template.template_text)
log_entry = LogEntry(
device_id=device.id,
template_id=template.id,
template_variables=json.dumps(variables) if variables else None,
severity=severity,
timestamp=datetime.utcnow()
)
template.usage_count = 1
compression_info = {
'used_template': True,
'template_alias': template.alias,
'new_template': True,
'original_size': len(message.encode('utf-8')),
'compressed_size': len(template.alias.encode('utf-8'))
}
else:
# Store as full message
log_entry = LogEntry(
device_id=device.id,
full_message=message,
severity=severity,
timestamp=datetime.utcnow()
)
compression_info = {
'used_template': False,
'stored_full': True,
'original_size': len(message.encode('utf-8'))
}
session.add(log_entry)
log_entry, compression_info = self.store_log_in_session(
session, device, message, severity
)
session.flush() # Get the log entry ID
return {
'success': True,
'log_id': log_entry.id,
@@ -159,7 +115,73 @@ class LogCompressionService:
'error': str(e),
'message': 'Log processing failed'
}
def store_log_in_session(self, session: Session, device: Device,
message: str, severity: str = 'info') -> Tuple[LogEntry, Dict]:
"""Compress and store a single log message using an existing session/device.
Shared by process_log_message() and bulk file ingestion so callers don't
open a new database session per log line. The caller is responsible for
committing (the session context manager does this).
Returns (log_entry, compression_info).
"""
# Try to match message to existing template
template, variables = self._match_message_template(session, message)
if template:
log_entry = LogEntry(
device_id=device.id,
template_id=template.id,
template_variables=json.dumps(variables) if variables else None,
severity=severity,
timestamp=datetime.utcnow()
)
template.usage_count += 1
original_size = len(message.encode('utf-8'))
compressed_size = len(template.alias.encode('utf-8')) + \
len(json.dumps(variables or {}).encode('utf-8'))
compression_info = {
'used_template': True,
'template_alias': template.alias,
'original_size': original_size,
'compressed_size': compressed_size,
'savings_percent': ((original_size - compressed_size) / original_size) * 100
}
else:
template, variables = self._create_new_template(session, message)
if template:
log_entry = LogEntry(
device_id=device.id,
template_id=template.id,
template_variables=json.dumps(variables) if variables else None,
severity=severity,
timestamp=datetime.utcnow()
)
template.usage_count = 1
compression_info = {
'used_template': True,
'template_alias': template.alias,
'new_template': True,
'original_size': len(message.encode('utf-8')),
'compressed_size': len(template.alias.encode('utf-8'))
}
else:
log_entry = LogEntry(
device_id=device.id,
full_message=message,
severity=severity,
timestamp=datetime.utcnow()
)
compression_info = {
'used_template': False,
'stored_full': True,
'original_size': len(message.encode('utf-8'))
}
session.add(log_entry)
return log_entry, compression_info
@staticmethod
def _infer_device_type(hostname: str) -> str:
"""Guess device type from hostname pattern."""
@@ -250,49 +272,37 @@ class LogCompressionService:
template_text=template_text,
category=pattern_info['category']
).first()
if template:
# Extract variables
variables = {}
for i, group in enumerate(match.groups(), 1):
# Map to variable names based on template
if '{card_id}' in template_text and pattern_info['category'] == 'card_detection':
variables['card_id'] = group
elif '{error}' in template_text and pattern_info['category'] == 'connection_error':
variables['error'] = group
elif '{time}' in template_text and pattern_info['category'] == 'system_startup':
variables['time'] = group
elif '{message}' in template_text:
variables['message'] = group
elif '{command}' in template_text and i == 1:
variables['command'] = group
elif '{status}' in template_text and i == 2:
variables['status'] = group
elif '{temp}' in template_text:
variables['temp'] = group
variables = self._variables_from_match(pattern_info, match)
return template, variables
return None, None
def _create_new_template(self, session: Session, message: str) -> Optional[MessageTemplate]:
"""Create new template if message matches a known pattern"""
def _create_new_template(self, session: Session, message: str) -> Tuple[Optional[MessageTemplate], Optional[Dict]]:
"""Create new template if message matches a known pattern.
Returns (template, variables) so the caller can persist the extracted
variables on the first log entry that creates the template.
"""
for pattern_info in self.template_patterns:
match = re.match(pattern_info['pattern'], message)
if match:
variables = self._variables_from_match(pattern_info, match)
# Check if template already exists
existing = session.query(MessageTemplate).filter_by(
template_text=pattern_info['template'],
category=pattern_info['category']
).first()
if existing:
return existing
return existing, variables
# Create new template
alias = self._generate_alias(session, pattern_info['alias_prefix'])
template_hash = MessageTemplate.create_hash(pattern_info['template'])
template = MessageTemplate(
template_hash=template_hash,
template_text=pattern_info['template'],
@@ -303,9 +313,9 @@ class LogCompressionService:
session.add(template)
session.flush()
return template
return template, variables
return None
return None, None
def _generate_alias(self, session: Session, prefix: str) -> str:
"""Generate unique alias for template"""
@@ -325,12 +335,24 @@ class LogCompressionService:
return f"{prefix}{max_num + 1:03d}"
def _extract_variables(self, message: str, template: str) -> Dict:
"""Extract variables from message using template"""
# Simple variable extraction - could be enhanced
variables = {}
# This is a simplified implementation
# In production, you'd want more sophisticated template matching
return variables
"""Extract template variables from a raw message.
Finds the matching pattern for the message and maps its capture groups
to the configured variable names. Returns {} when no pattern matches.
"""
for pattern_info in self.template_patterns:
if pattern_info['template'] != template:
continue
match = re.match(pattern_info['pattern'], message)
if match:
return self._variables_from_match(pattern_info, match)
return {}
@staticmethod
def _variables_from_match(pattern_info: Dict, match) -> Dict:
"""Map a regex match's capture groups to named template variables."""
names = pattern_info.get('variables') or []
return {name: value for name, value in zip(names, match.groups())}
def get_compression_stats(self) -> Dict:
"""Get compression statistics"""