add pgbouncer and use it to pool db connexions

Dockerfile

@@ -12,6 +12,7 @@ RUN apt-get update && apt-get install -y \
     build-essential \
     default-libmysqlclient-dev \
     libpq-dev \
+    pgbouncer \
     git \
     curl \
     && apt-get clean && rm -rf /var/lib/apt/lists/*
@@ -28,6 +29,11 @@ COPY --chown=airflow:airflow requirements.txt /requirements.txt
 COPY --chown=airflow:airflow dags $AIRFLOW_HOME/dags
 COPY --chown=airflow:airflow entrypoint.sh /entrypoint.sh
 
+# Create the pgbouncer configuration directory
+# and give ownership to airflow user
+RUN mkdir -p /etc/pgbouncer \
+    && chown -R airflow:airflow /etc/pgbouncer
+
 # Make entrypoint executable
 RUN chmod +x /entrypoint.sh
 

entrypoint.sh

@@ -1,15 +1,60 @@
 #!/bin/bash
 set -e
 
-# Charger les variables d'environnement
+# Load environment variables from .env file if it exists
+# This is useful for local development
 if [ -f .env ]; then
   export $(grep -v '^#' .env | xargs)
 fi
 
-# Init DB si besoin
+# Dynamically extract database connection details from environment variables
+# airflow_db
+AF_USER=$(echo "$AIRFLOW__CORE__SQL_ALCHEMY_CONN" | sed -E 's|postgresql\+psycopg2://([^:]+):.*|\1|')
+AF_PASS=$(echo "$AIRFLOW__CORE__SQL_ALCHEMY_CONN" | sed -E 's|postgresql\+psycopg2://[^:]+:([^@]+)@.*|\1|')
+AF_HOST=$(echo "$AIRFLOW__CORE__SQL_ALCHEMY_CONN" | sed -E 's|.*@([^:/]+):([0-9]+)/.*|\1|')
+AF_PORT=$(echo "$AIRFLOW__CORE__SQL_ALCHEMY_CONN" | sed -E 's|.*@[^:/]+:([0-9]+)/.*|\1|')
+AF_DB=$(echo "$AIRFLOW__CORE__SQL_ALCHEMY_CONN" | sed -E 's|.*/([^?]+).*|\1|')
+
+# transaction_db
+TX_USER=$(echo "$DATABASE_URL" | sed -E 's|postgresql\+psycopg2://([^:]+):.*|\1|')
+TX_PASS=$(echo "$DATABASE_URL" | sed -E 's|postgresql\+psycopg2://[^:]+:([^@]+)@.*|\1|')
+TX_DB=$(echo "$DATABASE_URL" | sed -E 's|.*/([^?]+).*|\1|')
+
+# Generate pgbouncer.ini
+cat <<EOF > /etc/pgbouncer/pgbouncer.ini
+[databases]
+airflow_db = host=${AF_HOST} port=${AF_PORT} dbname=${AF_DB} user=${AF_USER} password=${AF_PASS}
+transaction_db = host=${AF_HOST} port=${AF_PORT} dbname=${TX_DB} user=${TX_USER} password=${TX_PASS}
+
+[pgbouncer]
+listen_addr = 127.0.0.1
+listen_port = 6432
+auth_type = trust
+auth_file = /etc/pgbouncer/userlist.txt
+pool_mode = transaction
+server_tls_sslmode = require
+ignore_startup_parameters = extra_float_digits
+log_connections = 0
+log_disconnections = 0
+default_pool_size = 5
+EOF
+
+# Generate userlist.txt
+cat <<EOF > /etc/pgbouncer/userlist.txt
+"${AF_USER}" "${AF_PASS}"
+"${TX_USER}" "${TX_PASS}"
+EOF
+
+# Launch PgBouncer
+pgbouncer /etc/pgbouncer/pgbouncer.ini &
+
+# Dynamically modify the URL to PgBouncer
+export AIRFLOW__CORE__SQL_ALCHEMY_CONN=postgresql+psycopg2://${AF_USER}:${AF_PASS}@127.0.0.1:6432/airflow_db
+
+# Init DB if needed
 airflow db init
 
-# Créer un user admin si aucun n'existe
+# Create a superuser if it doesn't exist
 airflow users create \
   --username admin \
   --firstname Admin \
@@ -18,8 +63,8 @@ airflow users create \
   --email [email protected] \
   --password admin || true
 
-# Lancer le scheduler en background
+# Launch the scheduler
 airflow scheduler &
 
-# Lancer le webserver (en foreground, pour que le conteneur reste "vivant")
+# Launch the webserver
 exec airflow webserver --port 8088