Implement database connection pooling with context manager pattern
- Added DBUtils PooledDB for intelligent connection pooling - Created db_pool.py with lazy-initialized connection pool (max 20 connections) - Added db_connection_context() context manager for safe connection handling - Refactored all 19 database operations to use context manager pattern - Ensures proper connection cleanup and exception handling - Prevents connection exhaustion on POST requests - Added logging configuration for debugging Changes: - py_app/app/db_pool.py: New connection pool manager - py_app/app/logging_config.py: Centralized logging - py_app/app/__init__.py: Updated to use connection pool - py_app/app/routes.py: Refactored all DB operations to use context manager - py_app/app/settings.py: Updated settings handlers - py_app/requirements.txt: Added DBUtils dependency This solves the connection timeout issues experienced with the fgscan page.
This commit is contained in:
122
py_app/app/db_pool.py
Normal file
122
py_app/app/db_pool.py
Normal file
@@ -0,0 +1,122 @@
|
||||
"""
|
||||
Database Connection Pool Manager for MariaDB
|
||||
Provides connection pooling to prevent connection exhaustion
|
||||
"""
|
||||
|
||||
import os
|
||||
import mariadb
|
||||
from dbutils.pooled_db import PooledDB
|
||||
from flask import current_app
|
||||
from app.logging_config import get_logger
|
||||
|
||||
logger = get_logger('db_pool')
|
||||
|
||||
# Global connection pool instance
|
||||
_db_pool = None
|
||||
_pool_initialized = False
|
||||
|
||||
def get_db_pool():
|
||||
"""
|
||||
Get or create the database connection pool.
|
||||
Implements lazy initialization to ensure app context is available and config file exists.
|
||||
This function should only be called when needing a database connection,
|
||||
after the database config file has been created.
|
||||
"""
|
||||
global _db_pool, _pool_initialized
|
||||
|
||||
logger.debug("get_db_pool() called")
|
||||
|
||||
if _db_pool is not None:
|
||||
logger.debug("Pool already initialized, returning existing pool")
|
||||
return _db_pool
|
||||
|
||||
if _pool_initialized:
|
||||
# Already tried to initialize but failed - don't retry
|
||||
logger.error("Pool initialization flag set but _db_pool is None - not retrying")
|
||||
raise RuntimeError("Database pool initialization failed previously")
|
||||
|
||||
try:
|
||||
logger.info("Initializing database connection pool...")
|
||||
|
||||
# Read settings from the configuration file
|
||||
settings_file = os.path.join(current_app.instance_path, 'external_server.conf')
|
||||
logger.debug(f"Looking for config file: {settings_file}")
|
||||
|
||||
if not os.path.exists(settings_file):
|
||||
raise FileNotFoundError(f"Database config file not found: {settings_file}")
|
||||
|
||||
logger.debug("Config file found, parsing...")
|
||||
settings = {}
|
||||
with open(settings_file, 'r') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
if '=' in line:
|
||||
key, value = line.split('=', 1)
|
||||
settings[key] = value
|
||||
|
||||
logger.debug(f"Parsed config: host={settings.get('server_domain')}, db={settings.get('database_name')}, user={settings.get('username')}")
|
||||
|
||||
# Validate we have all required settings
|
||||
required_keys = ['username', 'password', 'server_domain', 'port', 'database_name']
|
||||
for key in required_keys:
|
||||
if key not in settings:
|
||||
raise ValueError(f"Missing database configuration: {key}")
|
||||
|
||||
logger.info(f"Creating connection pool: max_connections=20, min_cached=3, max_cached=10, max_shared=5")
|
||||
|
||||
# Create connection pool
|
||||
_db_pool = PooledDB(
|
||||
creator=mariadb,
|
||||
maxconnections=20, # Max connections in pool
|
||||
mincached=3, # Min idle connections
|
||||
maxcached=10, # Max idle connections
|
||||
maxshared=5, # Shared connections
|
||||
blocking=True, # Block if no connection available
|
||||
ping=1, # Ping database to check connection health (1 = on demand)
|
||||
user=settings['username'],
|
||||
password=settings['password'],
|
||||
host=settings['server_domain'],
|
||||
port=int(settings['port']),
|
||||
database=settings['database_name'],
|
||||
autocommit=False # Explicit commit for safety
|
||||
)
|
||||
|
||||
_pool_initialized = True
|
||||
logger.info("✅ Database connection pool initialized successfully (max 20 connections)")
|
||||
return _db_pool
|
||||
|
||||
except Exception as e:
|
||||
_pool_initialized = True
|
||||
logger.error(f"FAILED to initialize database pool: {e}", exc_info=True)
|
||||
raise RuntimeError(f"Database pool initialization failed: {e}") from e
|
||||
|
||||
def get_db_connection():
|
||||
"""
|
||||
Get a connection from the pool.
|
||||
Always use with 'with' statement or ensure close() is called.
|
||||
"""
|
||||
logger.debug("get_db_connection() called")
|
||||
try:
|
||||
pool = get_db_pool()
|
||||
conn = pool.connection()
|
||||
logger.debug("Successfully obtained connection from pool")
|
||||
return conn
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get connection from pool: {e}", exc_info=True)
|
||||
raise
|
||||
|
||||
def close_db_pool():
|
||||
"""
|
||||
Close all connections in the pool (called at app shutdown).
|
||||
"""
|
||||
global _db_pool
|
||||
if _db_pool:
|
||||
logger.info("Closing database connection pool...")
|
||||
_db_pool.close()
|
||||
_db_pool = None
|
||||
logger.info("✅ Database connection pool closed")
|
||||
|
||||
# That's it! The pool is lazily initialized on first connection.
|
||||
# No other initialization needed.
|
||||
Reference in New Issue
Block a user