Initial commit — Server_Monitorizare_v2
This commit is contained in:
ske087
256
app/services/file_service.py
Normal file
256
app/services/file_service.py
Normal file
@@ -0,0 +1,256 @@
|
||||
"""
|
||||
File upload and processing service
|
||||
"""
|
||||
import os
|
||||
import hashlib
|
||||
import mimetypes
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from werkzeug.utils import secure_filename
|
||||
from app.models import Device, FileUpload
|
||||
from config.database_config import get_db
|
||||
import logging
|
||||
|
||||
class FileUploadService:
|
||||
"""Service for handling file uploads and processing"""
|
||||
|
||||
def __init__(self):
|
||||
self.db = get_db()
|
||||
self.upload_folder = Path("data/uploads")
|
||||
self.upload_folder.mkdir(exist_ok=True)
|
||||
|
||||
# Allowed file extensions
|
||||
self.allowed_extensions = {
|
||||
'txt', 'log', 'conf', 'cfg', 'json', 'yml', 'yaml',
|
||||
'py', 'sh', 'service', 'env', 'ini'
|
||||
}
|
||||
|
||||
# Max file size (50MB)
|
||||
self.max_file_size = 50 * 1024 * 1024
|
||||
|
||||
def process_uploaded_file(self, file, device_info):
|
||||
"""Process uploaded file from device"""
|
||||
try:
|
||||
# Validate file
|
||||
if not file or file.filename == '':
|
||||
return {'success': False, 'error': 'No file provided'}
|
||||
|
||||
# Check file extension
|
||||
filename = secure_filename(file.filename)
|
||||
if not self._allowed_file(filename):
|
||||
return {
|
||||
'success': False,
|
||||
'error': f'File type not allowed. Allowed: {", ".join(self.allowed_extensions)}'
|
||||
}
|
||||
|
||||
# Check file size
|
||||
file.seek(0, 2) # Seek to end
|
||||
file_size = file.tell()
|
||||
file.seek(0) # Reset position
|
||||
|
||||
if file_size > self.max_file_size:
|
||||
return {
|
||||
'success': False,
|
||||
'error': f'File too large. Max size: {self.max_file_size // (1024*1024)}MB'
|
||||
}
|
||||
|
||||
# Calculate file hash
|
||||
file_content = file.read()
|
||||
file.seek(0) # Reset for saving
|
||||
file_hash = hashlib.sha256(file_content).hexdigest()
|
||||
|
||||
with self.db.get_session() as session:
|
||||
# Get or create device
|
||||
device = self._get_or_create_device(session, device_info)
|
||||
|
||||
# Check for duplicate file
|
||||
existing_file = session.query(FileUpload).filter_by(file_hash=file_hash).first()
|
||||
if existing_file:
|
||||
return {
|
||||
'success': True,
|
||||
'message': 'File already exists (duplicate detected)',
|
||||
'file_id': existing_file.id,
|
||||
'duplicate': True
|
||||
}
|
||||
|
||||
# Generate unique filename
|
||||
timestamp = datetime.utcnow().strftime("%Y%m%d_%H%M%S")
|
||||
new_filename = f"{device.hostname}_{timestamp}_{filename}"
|
||||
file_path = self.upload_folder / new_filename
|
||||
|
||||
# Save file
|
||||
with open(file_path, 'wb') as f:
|
||||
f.write(file_content)
|
||||
|
||||
# Get MIME type
|
||||
mime_type, _ = mimetypes.guess_type(filename)
|
||||
|
||||
# Create file record
|
||||
file_upload = FileUpload(
|
||||
device_id=device.id,
|
||||
filename=new_filename,
|
||||
original_filename=filename,
|
||||
file_path=str(file_path),
|
||||
file_size=file_size,
|
||||
file_hash=file_hash,
|
||||
mime_type=mime_type,
|
||||
upload_date=datetime.utcnow(),
|
||||
upload_ip=device_info.get('device_ip'),
|
||||
processed=False,
|
||||
processing_status='pending'
|
||||
)
|
||||
|
||||
session.add(file_upload)
|
||||
session.flush()
|
||||
|
||||
# Process file content if it's a log file
|
||||
if self._is_log_file(filename, mime_type):
|
||||
self._process_log_file(file_upload, file_content)
|
||||
|
||||
return {
|
||||
'success': True,
|
||||
'message': 'File uploaded successfully',
|
||||
'file_id': file_upload.id,
|
||||
'filename': new_filename,
|
||||
'size': file_size,
|
||||
'hash': file_hash,
|
||||
'processed': file_upload.processed
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error processing uploaded file: {e}")
|
||||
return {
|
||||
'success': False,
|
||||
'error': str(e)
|
||||
}
|
||||
|
||||
def _allowed_file(self, filename):
|
||||
"""Check if file extension is allowed"""
|
||||
return '.' in filename and \
|
||||
filename.rsplit('.', 1)[1].lower() in self.allowed_extensions
|
||||
|
||||
def _get_or_create_device(self, session, device_info):
|
||||
"""Get existing device or create new one"""
|
||||
device = session.query(Device).filter_by(
|
||||
hostname=device_info['hostname'],
|
||||
device_ip=device_info['device_ip']
|
||||
).first()
|
||||
|
||||
if not device:
|
||||
device = Device(
|
||||
hostname=device_info['hostname'],
|
||||
device_ip=device_info['device_ip'],
|
||||
nume_masa=device_info['nume_masa'],
|
||||
last_seen=datetime.utcnow(),
|
||||
status='active'
|
||||
)
|
||||
session.add(device)
|
||||
session.flush()
|
||||
else:
|
||||
device.last_seen = datetime.utcnow()
|
||||
if device.nume_masa != device_info['nume_masa']:
|
||||
device.nume_masa = device_info['nume_masa']
|
||||
|
||||
return device
|
||||
|
||||
def _is_log_file(self, filename, mime_type):
|
||||
"""Check if file is a log file that should be processed"""
|
||||
log_extensions = {'log', 'txt'}
|
||||
log_keywords = ['log', 'error', 'debug', 'trace', 'audit']
|
||||
|
||||
# Check extension
|
||||
if '.' in filename:
|
||||
ext = filename.rsplit('.', 1)[1].lower()
|
||||
if ext in log_extensions:
|
||||
return True
|
||||
|
||||
# Check filename for log keywords
|
||||
filename_lower = filename.lower()
|
||||
for keyword in log_keywords:
|
||||
if keyword in filename_lower:
|
||||
return True
|
||||
|
||||
# Check MIME type
|
||||
if mime_type and 'text' in mime_type:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _process_log_file(self, file_upload, content):
|
||||
"""Process log file content to extract log entries"""
|
||||
try:
|
||||
# Mark as log file
|
||||
file_upload.is_log_file = True
|
||||
|
||||
# Simple log processing - split by lines
|
||||
lines = content.decode('utf-8', errors='ignore').split('\n')
|
||||
entries_extracted = 0
|
||||
|
||||
from app.services.log_service import LogCompressionService
|
||||
log_service = LogCompressionService()
|
||||
|
||||
device_info = {
|
||||
'hostname': file_upload.device.hostname,
|
||||
'device_ip': file_upload.device.device_ip,
|
||||
'nume_masa': file_upload.device.nume_masa
|
||||
}
|
||||
|
||||
for line_num, line in enumerate(lines, 1):
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
# Try to extract timestamp and message
|
||||
# This is a simple implementation - enhance as needed
|
||||
message = f"[File: {file_upload.original_filename}:{line_num}] {line}"
|
||||
|
||||
# Process through log compression service
|
||||
result = log_service.process_log_message(
|
||||
device_info=device_info,
|
||||
message=message,
|
||||
severity='info'
|
||||
)
|
||||
|
||||
if result['success']:
|
||||
entries_extracted += 1
|
||||
|
||||
# Update file record
|
||||
file_upload.log_entries_extracted = entries_extracted
|
||||
file_upload.processed = True
|
||||
file_upload.processing_status = 'completed'
|
||||
|
||||
logging.info(f"Processed log file {file_upload.filename}: {entries_extracted} entries extracted")
|
||||
|
||||
except Exception as e:
|
||||
logging.error(f"Error processing log file content: {e}")
|
||||
file_upload.processing_status = 'error'
|
||||
file_upload.processing_error = str(e)
|
||||
|
||||
def get_upload_stats(self):
|
||||
"""Get file upload statistics"""
|
||||
try:
|
||||
with self.db.get_session() as session:
|
||||
total_files = session.query(FileUpload).count()
|
||||
log_files = session.query(FileUpload).filter_by(is_log_file=True).count()
|
||||
|
||||
# Calculate total size
|
||||
total_size = session.query(
|
||||
session.func.sum(FileUpload.file_size)
|
||||
).scalar() or 0
|
||||
|
||||
# Count by processing status
|
||||
processed = session.query(FileUpload).filter_by(processed=True).count()
|
||||
pending = session.query(FileUpload).filter(
|
||||
FileUpload.processing_status == 'pending'
|
||||
).count()
|
||||
|
||||
return {
|
||||
'total_files': total_files,
|
||||
'log_files': log_files,
|
||||
'total_size': total_size,
|
||||
'processed': processed,
|
||||
'pending': pending
|
||||
}
|
||||
except Exception as e:
|
||||
logging.error(f"Error getting upload stats: {e}")
|
||||
return {'error': str(e)}
|
||||
Reference in New Issue
Block a user