64b67b2979
- Added DBUtils PooledDB for intelligent connection pooling - Created db_pool.py with lazy-initialized connection pool (max 20 connections) - Added db_connection_context() context manager for safe connection handling - Refactored all 19 database operations to use context manager pattern - Ensures proper connection cleanup and exception handling - Prevents connection exhaustion on POST requests - Added logging configuration for debugging Changes: - py_app/app/db_pool.py: New connection pool manager - py_app/app/logging_config.py: Centralized logging - py_app/app/__init__.py: Updated to use connection pool - py_app/app/routes.py: Refactored all DB operations to use context manager - py_app/app/settings.py: Updated settings handlers - py_app/requirements.txt: Added DBUtils dependency This solves the connection timeout issues experienced with the fgscan page.
123 lines
4.5 KiB
Python
123 lines
4.5 KiB
Python
"""
|
|
Database Connection Pool Manager for MariaDB
|
|
Provides connection pooling to prevent connection exhaustion
|
|
"""
|
|
|
|
import os
|
|
import mariadb
|
|
from dbutils.pooled_db import PooledDB
|
|
from flask import current_app
|
|
from app.logging_config import get_logger
|
|
|
|
logger = get_logger('db_pool')
|
|
|
|
# Global connection pool instance
|
|
_db_pool = None
|
|
_pool_initialized = False
|
|
|
|
def get_db_pool():
|
|
"""
|
|
Get or create the database connection pool.
|
|
Implements lazy initialization to ensure app context is available and config file exists.
|
|
This function should only be called when needing a database connection,
|
|
after the database config file has been created.
|
|
"""
|
|
global _db_pool, _pool_initialized
|
|
|
|
logger.debug("get_db_pool() called")
|
|
|
|
if _db_pool is not None:
|
|
logger.debug("Pool already initialized, returning existing pool")
|
|
return _db_pool
|
|
|
|
if _pool_initialized:
|
|
# Already tried to initialize but failed - don't retry
|
|
logger.error("Pool initialization flag set but _db_pool is None - not retrying")
|
|
raise RuntimeError("Database pool initialization failed previously")
|
|
|
|
try:
|
|
logger.info("Initializing database connection pool...")
|
|
|
|
# Read settings from the configuration file
|
|
settings_file = os.path.join(current_app.instance_path, 'external_server.conf')
|
|
logger.debug(f"Looking for config file: {settings_file}")
|
|
|
|
if not os.path.exists(settings_file):
|
|
raise FileNotFoundError(f"Database config file not found: {settings_file}")
|
|
|
|
logger.debug("Config file found, parsing...")
|
|
settings = {}
|
|
with open(settings_file, 'r') as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line or line.startswith('#'):
|
|
continue
|
|
if '=' in line:
|
|
key, value = line.split('=', 1)
|
|
settings[key] = value
|
|
|
|
logger.debug(f"Parsed config: host={settings.get('server_domain')}, db={settings.get('database_name')}, user={settings.get('username')}")
|
|
|
|
# Validate we have all required settings
|
|
required_keys = ['username', 'password', 'server_domain', 'port', 'database_name']
|
|
for key in required_keys:
|
|
if key not in settings:
|
|
raise ValueError(f"Missing database configuration: {key}")
|
|
|
|
logger.info(f"Creating connection pool: max_connections=20, min_cached=3, max_cached=10, max_shared=5")
|
|
|
|
# Create connection pool
|
|
_db_pool = PooledDB(
|
|
creator=mariadb,
|
|
maxconnections=20, # Max connections in pool
|
|
mincached=3, # Min idle connections
|
|
maxcached=10, # Max idle connections
|
|
maxshared=5, # Shared connections
|
|
blocking=True, # Block if no connection available
|
|
ping=1, # Ping database to check connection health (1 = on demand)
|
|
user=settings['username'],
|
|
password=settings['password'],
|
|
host=settings['server_domain'],
|
|
port=int(settings['port']),
|
|
database=settings['database_name'],
|
|
autocommit=False # Explicit commit for safety
|
|
)
|
|
|
|
_pool_initialized = True
|
|
logger.info("✅ Database connection pool initialized successfully (max 20 connections)")
|
|
return _db_pool
|
|
|
|
except Exception as e:
|
|
_pool_initialized = True
|
|
logger.error(f"FAILED to initialize database pool: {e}", exc_info=True)
|
|
raise RuntimeError(f"Database pool initialization failed: {e}") from e
|
|
|
|
def get_db_connection():
|
|
"""
|
|
Get a connection from the pool.
|
|
Always use with 'with' statement or ensure close() is called.
|
|
"""
|
|
logger.debug("get_db_connection() called")
|
|
try:
|
|
pool = get_db_pool()
|
|
conn = pool.connection()
|
|
logger.debug("Successfully obtained connection from pool")
|
|
return conn
|
|
except Exception as e:
|
|
logger.error(f"Failed to get connection from pool: {e}", exc_info=True)
|
|
raise
|
|
|
|
def close_db_pool():
|
|
"""
|
|
Close all connections in the pool (called at app shutdown).
|
|
"""
|
|
global _db_pool
|
|
if _db_pool:
|
|
logger.info("Closing database connection pool...")
|
|
_db_pool.close()
|
|
_db_pool = None
|
|
logger.info("✅ Database connection pool closed")
|
|
|
|
# That's it! The pool is lazily initialized on first connection.
|
|
# No other initialization needed.
|