diff --git a/.gitignore b/.gitignore index cdf645256..0abb22182 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,10 @@ htmlcov .pytest_cache local_blob_storage services/coverage.xml + +resources/nginx/ssl/*.crt +resources/nginx/ssl/*.key +resources/nginx/ssl/*.pem +services/resources/ca/*.crt +services/resources/ca/*.key +services/resources/ca/*.pem \ No newline at end of file diff --git a/docker-compose-addons.yml b/docker-compose-addons.yml index ef59efd39..4f7c560de 100644 --- a/docker-compose-addons.yml +++ b/docker-compose-addons.yml @@ -27,6 +27,8 @@ services: DESTINATION_DB: ${COUNT_DESTINATION_DB} + PG_SSL_REQUIRED: ${PG_SSL_REQUIRED:-True} + MONGO_DB_URI: ${MONGO_DB_URI} MONGO_DB_NAME: ${CM_DATABASE_NAME} INPUT_COUNTS_MONGO_COLLECTION_NAME: ${INPUT_COUNTS_MONGO_COLLECTION_NAME} @@ -35,6 +37,8 @@ services: KAFKA_BIGQUERY_TABLENAME: ${KAFKA_BIGQUERY_TABLENAME} LOGGING_LEVEL: ${COUNTS_LOGGING_LEVEL} + volumes: + - ./services/resources/ca:/etc/ssl/ca logging: options: max-size: '10m' @@ -67,7 +71,11 @@ services: PG_DB_USER: ${PG_DB_USER} PG_DB_PASS: ${PG_DB_PASS} + PG_SSL_REQUIRED: ${PG_SSL_REQUIRED:-True} + LOGGING_LEVEL: ${RSU_STATUS_LOGGING_LEVEL} + volumes: + - ./services/resources/ca:/etc/ssl/ca logging: options: max-size: '10m' @@ -82,97 +90,97 @@ services: dockerfile: Dockerfile.iss_health_check image: iss_health_check:latest restart: on-failure:3 - depends_on: - cvmanager_postgres: - condition: service_healthy - required: false environment: - STORAGE_TYPE: ${STORAGE_TYPE} + STORAGE_TYPE: ${STORAGE_TYPE:-POSTGRES} ISS_API_KEY: ${ISS_API_KEY} ISS_API_KEY_NAME: ${ISS_API_KEY_NAME} ISS_PROJECT_ID: ${ISS_PROJECT_ID} ISS_SCMS_TOKEN_REST_ENDPOINT: ${ISS_SCMS_TOKEN_REST_ENDPOINT} ISS_SCMS_VEHICLE_REST_ENDPOINT: ${ISS_SCMS_VEHICLE_REST_ENDPOINT} - ISS_KEY_TABLE_NAME: ${ISS_KEY_TABLE_NAME} + ISS_KEY_TABLE_NAME: ${ISS_KEY_TABLE_NAME:-iss_keys} PG_DB_HOST: ${PG_DB_HOST} PG_DB_NAME: ${PG_DB_NAME} PG_DB_USER: ${PG_DB_USER} PG_DB_PASS: ${PG_DB_PASS} + PG_SSL_REQUIRED: ${PG_SSL_REQUIRED:-True} + PROJECT_ID: ${GCP_PROJECT_ID} GOOGLE_APPLICATION_CREDENTIALS: /home/gcp_key.json LOGGING_LEVEL: ${ISS_LOGGING_LEVEL} volumes: - ./resources/google/${GOOGLE_ACCESS_KEY_NAME}:/home/gcp_key.json + - ./services/resources/ca:/etc/ssl/ca logging: options: max-size: '10m' max-file: '5' - firmware_manager_upgrade_scheduler: - profiles: - - addons - - firmware_manager_upgrade_scheduler - build: - context: services - dockerfile: Dockerfile.fmus - image: jpo_firmware_manager_upgrade_scheduler:latest - restart: on-failure:3 - - ports: - - '8089:8080' - environment: - PG_DB_HOST: ${PG_DB_HOST} - PG_DB_NAME: postgres - PG_DB_USER: ${PG_DB_USER} - PG_DB_PASS: ${PG_DB_PASS} - - UPGRADE_RUNNER_ENDPOINT: ${FIRMWARE_MANAGER_UPGRADE_RUNNER_ENDPOINT} - - LOGGING_LEVEL: ${FIRMWARE_MANAGER_LOGGING_LEVEL} - volumes: - - ${HOST_BLOB_STORAGE_DIRECTORY}:/mnt/blob_storage - logging: - options: - max-size: '10m' - max-file: '5' - - firmware_manager_upgrade_runner: - profiles: - - addons - - firmware_manager_upgrade_runner - build: - context: services - dockerfile: Dockerfile.fmur - image: jpo_firmware_manager_upgrade_runner:latest - restart: on-failure:3 - - ports: - - '8090:8080' - environment: - BLOB_STORAGE_PROVIDER: ${BLOB_STORAGE_PROVIDER} - BLOB_STORAGE_BUCKET: ${BLOB_STORAGE_BUCKET} - - FW_UPGRADE_MAX_RETRY_LIMIT: ${FW_UPGRADE_MAX_RETRY_LIMIT} - - GCP_PROJECT: ${GCP_PROJECT_ID} - GOOGLE_APPLICATION_CREDENTIALS: /home/gcp_key.json - - SMTP_SERVER_IP: ${SMTP_SERVER_IP} - SMTP_EMAIL: ${SMTP_EMAIL} - SMTP_USERNAME: ${SMTP_USERNAME} - SMTP_PASSWORD: ${SMTP_PASSWORD} - - UPGRADE_SCHEDULER_ENDPOINT: ${FIRMWARE_MANAGER_UPGRADE_SCHEDULER_ENDPOINT} - - LOGGING_LEVEL: ${FIRMWARE_MANAGER_LOGGING_LEVEL} - volumes: - - ./resources/google/${GOOGLE_ACCESS_KEY_NAME}:/home/gcp_key.json - - ${HOST_BLOB_STORAGE_DIRECTORY}:/mnt/blob_storage - logging: - options: - max-size: '10m' - max-file: '5' +# firmware_manager_upgrade_scheduler: +# profiles: +# - addons +# - firmware_manager_upgrade_scheduler +# build: +# context: services +# dockerfile: Dockerfile.fmus +# image: jpo_firmware_manager_upgrade_scheduler:latest +# restart: on-failure:3 +# +# ports: +# - '8089:8080' +# environment: +# PG_DB_HOST: ${PG_DB_HOST} +# PG_DB_NAME: postgres +# PG_DB_USER: ${PG_DB_USER} +# PG_DB_PASS: ${PG_DB_PASS} +# +# UPGRADE_RUNNER_ENDPOINT: ${FIRMWARE_MANAGER_UPGRADE_RUNNER_ENDPOINT} +# +# LOGGING_LEVEL: ${FIRMWARE_MANAGER_LOGGING_LEVEL} +# volumes: +# - ${HOST_BLOB_STORAGE_DIRECTORY}:/mnt/blob_storage +# logging: +# options: +# max-size: '10m' +# max-file: '5' +# +# firmware_manager_upgrade_runner: +# profiles: +# - addons +# - firmware_manager_upgrade_runner +# build: +# context: services +# dockerfile: Dockerfile.fmur +# image: jpo_firmware_manager_upgrade_runner:latest +# restart: on-failure:3 +# +# ports: +# - '8090:8080' +# environment: +# BLOB_STORAGE_PROVIDER: ${BLOB_STORAGE_PROVIDER} +# BLOB_STORAGE_BUCKET: ${BLOB_STORAGE_BUCKET} +# +# FW_UPGRADE_MAX_RETRY_LIMIT: ${FW_UPGRADE_MAX_RETRY_LIMIT} +# +# GCP_PROJECT: ${GCP_PROJECT_ID} +# GOOGLE_APPLICATION_CREDENTIALS: /home/gcp_key.json +# +# SMTP_SERVER_IP: ${SMTP_SERVER_IP} +# SMTP_EMAIL: ${SMTP_EMAIL} +# SMTP_USERNAME: ${SMTP_USERNAME} +# SMTP_PASSWORD: ${SMTP_PASSWORD} +# +# UPGRADE_SCHEDULER_ENDPOINT: ${FIRMWARE_MANAGER_UPGRADE_SCHEDULER_ENDPOINT} +# +# LOGGING_LEVEL: ${FIRMWARE_MANAGER_LOGGING_LEVEL} +# volumes: +# - ./resources/google/${GOOGLE_ACCESS_KEY_NAME}:/home/gcp_key.json +# - ${HOST_BLOB_STORAGE_DIRECTORY}:/mnt/blob_storage +# - ${NGINX_PROXY_RESOURCES:-./resources/nginx}/ssl/server.crt:/etc/ssl/certs/server.crt +# logging: +# options: +# max-size: '10m' +# max-file: '5' diff --git a/docker-compose.yml b/docker-compose.yml index bd06227bc..7b1bdd775 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,33 @@ include: - docker-compose-obu-ota-server.yml services: + cvmanager_nginx_proxy: + profiles: + - nginx_proxy + build: + context: ./resources/nginx + dockerfile: Dockerfile + image: cvmanager_nginx_proxy:latest + restart: unless-stopped + ports: + - 80:80 + - 443:443 + environment: + NGINX_ENVSUBST_OUTPUT_DIR: /etc/nginx + SERVER_HOST: ${WEBAPP_DOMAIN} + volumes: + - ./resources/nginx/nginx-ssl.conf:/etc/nginx/templates/nginx.conf.template + - ./resources/nginx/ssl:/etc/ssl + depends_on: + - cvmanager_webapp + - cvmanager_keycloak + - cvmanager_api + - cvmanager_postgres + logging: + options: + max-size: '10m' + max-file: '5' + cvmanager_api: profiles: - basic @@ -16,14 +43,11 @@ services: restart: always extra_hosts: ${WEBAPP_DOMAIN}: ${WEBAPP_HOST_IP} - ${KEYCLOAK_DOMAIN}: ${KC_HOST_IP} - ports: - - '8081:5000' environment: PG_DB_HOST: ${PG_DB_HOST} PG_DB_USER: ${PG_DB_USER} PG_DB_PASS: ${PG_DB_PASS} - PG_DB_NAME: postgres + PG_DB_NAME: ${PG_DB_NAME} INSTANCE_CONNECTION_NAME: ${INSTANCE_CONNECTION_NAME} MONGO_DB_URI: ${MONGO_DB_URI} @@ -73,8 +97,14 @@ services: TIMEZONE: ${TIMEZONE} LOGGING_LEVEL: ${API_LOGGING_LEVEL} + KEYCLOAK_VERIFY: ${KEYCLOAK_VERIFY:-/etc/ssl/ca/ca.crt} + PG_SSL_REQUIRED: ${PG_SSL_REQUIRED:-True} + REQUESTS_CA_BUNDLE: /etc/ssl/ca/ca.crt + depends_on: + - cvmanager_postgres volumes: - ./resources/google/${GOOGLE_ACCESS_KEY_NAME}:/home/gcp_key.json + - ./services/resources/ca:/etc/ssl/ca logging: options: max-size: '10m' @@ -119,9 +149,6 @@ services: required: false extra_hosts: ${WEBAPP_DOMAIN}: ${WEBAPP_HOST_IP} - ${KEYCLOAK_DOMAIN}: ${KC_HOST_IP} - ports: - - '80:80' logging: options: max-size: '10m' @@ -162,13 +189,8 @@ services: cvmanager_postgres: required: false condition: service_started - extra_hosts: ${WEBAPP_DOMAIN}: ${WEBAPP_HOST_IP} - ${KEYCLOAK_DOMAIN}: ${KC_HOST_IP} - ports: - - '8084:8080' - - '9000:9000' environment: KEYCLOAK_ADMIN: ${KEYCLOAK_ADMIN} KEYCLOAK_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD} @@ -176,17 +198,21 @@ services: KC_BOOTSTRAP_ADMIN_USERNAME: ${KEYCLOAK_ADMIN} KC_BOOTSTRAP_ADMIN_PASSWORD: ${KEYCLOAK_ADMIN_PASSWORD} KC_HEALTH_ENABLED: 'true' - KC_LOG_LEVEL: ${KC_LOGGING_LEVEL} + KC_LOG_LEVEL: ${KC_LOGGING_LEVEL:-INFO} KC_DB: postgres KC_DB_URL: jdbc:postgresql://${PG_DB_HOST}/postgres?currentSchema=keycloak KC_DB_USERNAME: ${PG_DB_USER} KC_DB_PASSWORD: ${PG_DB_PASS} KC_HOSTNAME: ${KEYCLOAK_DOMAIN} + KC_HOSTNAME_PORT: 443 + KC_HOSTNAME_STRICT_HTTPS: 'true' + KC_HOSTNAME_STRICT: 'false' KEYCLOAK_API_CLIENT_SECRET_KEY: ${KEYCLOAK_API_CLIENT_SECRET_KEY} GOOGLE_CLIENT_ID: ${GOOGLE_CLIENT_ID} GOOGLE_CLIENT_SECRET: ${GOOGLE_CLIENT_SECRET} - KC_HOSTNAME_STRICT_HTTPS: 'false' KC_HTTP_ENABLED: 'true' + KC_PROXY_HEADERS: xforwarded + KC_HOSTNAME_STRICT_BACKCHANNEL: 'false' command: - start-dev - --import-realm @@ -194,7 +220,7 @@ services: options: max-size: '10m' healthcheck: - test: ['CMD', 'curl', '-f', 'http://localhost:9000/health'] + test: ['CMD', 'curl', '-f', 'http://localhost:8080/realms/master'] interval: 30s timeout: 10s retries: 3 diff --git a/resources/keycloak/realm.json b/resources/keycloak/realm.json index 36fe2dfdd..f6e7198ea 100644 --- a/resources/keycloak/realm.json +++ b/resources/keycloak/realm.json @@ -27,7 +27,7 @@ "oauth2DeviceCodeLifespan": 600, "oauth2DevicePollingInterval": 5, "enabled": true, - "sslRequired": "external", + "sslRequired": "all", "registrationAllowed": false, "registrationEmailAsUsername": false, "rememberMe": false, @@ -688,8 +688,8 @@ "enabled": true, "alwaysDisplayInConsole": false, "clientAuthenticatorType": "client-secret", - "redirectUris": ["http://localhost:3000/*", "http://localhost/*", "http://cvmanager.local.com/*"], - "webOrigins": ["http://localhost:3000", "http://localhost", "http://cvmanager.local.com"], + "redirectUris": ["http://localhost:3000/*", "http://localhost/*", "https://cvmanager.local.com/*"], + "webOrigins": ["http://localhost:3000", "http://localhost", "https://cvmanager.local.com"], "notBefore": 0, "bearerOnly": false, "consentRequired": false, diff --git a/resources/nginx/Dockerfile b/resources/nginx/Dockerfile new file mode 100644 index 000000000..b02794b5b --- /dev/null +++ b/resources/nginx/Dockerfile @@ -0,0 +1,14 @@ +FROM nginx:alpine + +# The official nginx image supports environment variable substitution in templates +# placed in /etc/nginx/templates/*.template. +# We mount the config in docker-compose, but we can also copy it here as a default. + +# Ensure we have openssl for dhparam generation if needed by the script +RUN apk add --no-network --no-cache openssl || apk add --no-cache openssl + +# Ensure gen_dhparam.sh is executable when mounted +COPY gen_dhparam.sh /docker-entrypoint.d/gen_dhparam.sh +RUN chmod +x /docker-entrypoint.d/gen_dhparam.sh + +# The gen_dhparam.sh script will be executed by the entrypoint because it's in /docker-entrypoint.d/ diff --git a/resources/nginx/README-deployment.md b/resources/nginx/README-deployment.md new file mode 100644 index 000000000..bba9f62f4 --- /dev/null +++ b/resources/nginx/README-deployment.md @@ -0,0 +1,124 @@ +### Deployment Documentation for JPO CV Manager + +This documentation provides instructions for deploying the JPO CV Manager to a VM with a specific domain name and an external PostgreSQL database. This setup uses NGINX as an SSL proxy for routing and SSL termination. + +#### Prerequisites + +- **Docker and Docker Compose**: Ensure you have Docker installed and running on your VM. +- **External PostgreSQL Database**: A PostgreSQL instance (version 15+ recommended) accessible from the VM. +- **SSL Certificates**: You must provide valid SSL certificates for your domain (e.g., from a corporate CA or other provider). +- **DNS Record**: A DNS A record or CNAME pointing your domain name to the VM's public IP address. + +#### 2. Configure SSL and CA Certificates + +The NGINX configuration expects the provided SSL certificates in the `resources/nginx/ssl` directory. + +1. Navigate to the NGINX resources directory: + ```bash + cd resources/nginx + mkdir -p ssl + ``` +2. Place your provided `server.crt` and `server.key` files in the `ssl` directory. + - `resources/nginx/ssl/server.crt` + - `resources/nginx/ssl/server.key` + +##### Adding a Custom CA Certificate (Optional) + +If your SSL certificate is issued by a private or corporate CA that is not in the standard trusted root list, you may need to add the CA certificate to the API service's truststore. This ensures the API can securely communicate with Keycloak. + +1. Create the CA resources directory: + ```bash + mkdir -p services/resources/ca + ``` +2. Place your CA certificate (in `.crt` format) in this directory as `ca.crt`: + - `services/resources/ca/ca.crt` +3. Rebuild the API service: + ```bash + docker compose build cvmanager_api + ``` + +Alternatively, you can mount the CA certificate at runtime by updating the `volumes` section of the `cvmanager_api` service in `docker-compose.yml`: + +```yaml + volumes: + - ./path/to/your-ca.crt:/etc/ssl/ca/ca.crt +``` + +#### 3. Environment Configuration + +Create or update your `.env` file in the project root. You can use `sample.env` as a template. Configure the following key variables for deployment: + +##### General and Domain Settings +```env +# The domain name of your deployment +WEBAPP_DOMAIN=cvmanager.example.com +KEYCLOAK_DOMAIN=cvmanager.example.com + +# The public IP or internal IP of the VM where Docker is running +DOCKER_HOST_IP= +WEBAPP_HOST_IP=${DOCKER_HOST_IP} + +# NGINX Resources path +NGINX_PROXY_RESOURCES=./resources/nginx +``` + +##### External PostgreSQL Settings +Configure these variables to point to your external database instead of the included container: +```env +# Connection details for the external Postgres instance +PG_DB_HOST=: +PG_DB_NAME= +PG_DB_USER= +PG_DB_PASS='' + +# Enable SSL for the connection to the external database +PG_SSL_REQUIRED=True +``` + +##### Service Endpoints +Ensure the endpoints use the configured domain: +```env +KEYCLOAK_ENDPOINT=https://${WEBAPP_DOMAIN} +WEBAPP_ENDPOINT=https://${WEBAPP_DOMAIN} +API_ENDPOINT=https://${WEBAPP_DOMAIN} +``` + +#### 3. Update Docker Profiles + +In your `.env` file, update the `COMPOSE_PROFILES` to include the necessary services but exclude the local postgres if you don't want it to run. However, since many services depend on the `basic` profile which includes `cvmanager_api` and `cvmanager_keycloak`, it is recommended to use specific service profiles or be aware that the `cvmanager_postgres` container may still start if the `basic` profile is used. + +To run only the necessary services for an external DB deployment, you can use: +```env +COMPOSE_PROFILES=nginx_proxy,cvmanager_api,cvmanager_keycloak,webapp,addons +``` + +#### 4. Run the Deployment + +Once the `.env` file and SSL certificates are in place, start the services using Docker Compose: + +```bash +# From the project root +docker compose up -d --build +``` + +The `cvmanager_nginx_proxy` service will: +1. Generate `dhparam.pem` if it's missing. +2. Substitute environment variables in the `nginx-ssl.conf` template. +3. Start NGINX on ports 80 and 443. + +#### 5. Verification + +1. **Browser Access**: Navigate to `https://cvmanager.example.com`. You should see the CV Manager login page. +2. **SSL**: Verify that the SSL certificate is valid and issued to your domain. +3. **Database Connection**: Check the logs for `cvmanager_api` and `cvmanager_keycloak` to ensure they successfully connected to the external database: + ```bash + docker compose logs cvmanager_api + docker compose logs cvmanager_keycloak + ``` +4. **Keycloak**: Ensure you can log in. Keycloak will automatically create its schema in the external database on the first run (using the `keycloak` schema if configured in the JDBC URL). + +#### Troubleshooting + +- **Database Connectivity**: Ensure the VM's IP is whitelisted in your external PostgreSQL's `pg_hba.conf` and that any firewalls allow traffic on the Postgres port (usually 5432). +- **SSL Certificate Mounts**: If NGINX fails to start, verify that the files exist at `resources/nginx/ssl/server.crt` and `resources/nginx/ssl/server.key`. +- **Domain Resolution**: Ensure your DNS is correctly configured. You can test this using `ping cvmanager.example.com` from a remote machine. diff --git a/resources/nginx/README-testing.md b/resources/nginx/README-testing.md new file mode 100644 index 000000000..fef06b36d --- /dev/null +++ b/resources/nginx/README-testing.md @@ -0,0 +1,116 @@ +### Local Testing Documentation for NGINX SSL Proxy + +This documentation provides instructions for setting up and testing the NGINX SSL proxy locally. The NGINX proxy serves as the entry point for the JPO CV Manager, handling SSL termination and routing requests to the webapp, API, and Keycloak services. + +#### Prerequisites + +- **Docker and Docker Compose**: Ensure you have Docker installed and running on your machine. +- **OpenSSL**: Required for generating self-signed certificates. +- **Root/Administrator Access**: Needed to modify the local `hosts` file. + +#### 1. Generate Self-Signed Certificates + +The NGINX configuration expects SSL certificates in the `resources/nginx/ssl` directory. You can generate these for local testing using the following commands: + +```bash +# Navigate to the nginx resources directory +cd resources/nginx + +# Create the ssl directory if it doesn't exist +mkdir -p ssl + +# Generate a self-signed certificate and private key with SAN for the web domain +openssl req -x509 -nodes -days 365 -newkey rsa:2048 \ + -keyout ssl/server.key -out ssl/server.crt \ + -subj "/C=US/ST=State/L=City/O=Organization/OU=Unit/CN=cvmanager.local.com" \ + -addext "subjectAltName = DNS:cvmanager.local.com" + +# Move certificate and key files to correct locations +mkdir -p ssl/certs ssl/private +mv ssl/server.crt ssl/certs/server.crt +mv ssl/server.key ssl/private/server.key + +# Copy the server certificate to the API resources CA directory to be trusted by the API +mkdir -p ../../services/resources/ca +cp ssl/server.crt ../../services/resources/ca/ca.crt +``` + +*Note: Ensure the `CN` (Common Name) and `subjectAltName` match the `WEBAPP_DOMAIN` defined in your `.env` file. Modern browsers and libraries (like Python's `requests`) require the `subjectAltName` extension for proper hostname verification.* + +#### 2. Configure Local Domain + +To access the services using the configured domain name (e.g., `cvmanager.local.com`), you need to update your local `hosts` file. + +**For Linux/macOS:** +Add the following line to `/etc/hosts`: +``` + cvmanager.local.com +``` + +**For Windows:** +Add the following line to `C:\Windows\System32\drivers\etc\hosts` (run Notepad as Administrator): +``` + cvmanager.local.com +``` + +*Note: Replace `` with your actual WSL2 IP address. You can find this by running `hostname -I` in your WSL terminal.* + +#### 3. Environment Configuration + +Ensure your `.env` file is properly configured. You can use `sample.env` as a template. Key variables for the NGINX setup include: + +```env +DOCKER_HOST_IP= +WEBAPP_DOMAIN=cvmanager.local.com +WEBAPP_HOST_IP=${DOCKER_HOST_IP} +NGINX_PROXY_RESOURCES=./resources/nginx +PG_SSL_REQUIRED=False +``` + +#### 4. Run the Setup + +To test the NGINX proxy, you only need to start the NGINX service along with its dependencies (WebApp, API, Keycloak, and Postgres). Use the following command to start the necessary services: + +```bash +# From the project root +docker compose --profile nginx_proxy --profile basic --profile webapp up --build +``` + +*Note: The `basic` profile includes `cvmanager_api`, `cvmanager_keycloak`, and `cvmanager_postgres`.* + +Alternatively, if you want to run everything: +```bash +docker compose up --build +``` + +The `cvmanager_nginx_proxy` service will: +1. Generate `dhparam.pem` using the `gen_dhparam.sh` script if it's missing. +2. Substitute environment variables in the `nginx-ssl.conf` template. +3. Start NGINX on ports 80 and 443. + +The `cvmanager_api` service is also configured to trust the generated `server.crt` certificate. This is necessary because the API communicates with Keycloak over HTTPS using the local domain name. Since the certificate is self-signed, it must be added to the API's truststore to avoid SSL verification errors. + +#### 5. Verification + +Once the services are up and running, you can verify the setup: + +- **HTTP Redirection**: Navigate to `http://cvmanager.local.com`. It should automatically redirect you to `https://cvmanager.local.com`. +- **HTTPS Access**: Navigate to `https://cvmanager.local.com`. Since you are using a self-signed certificate, your browser will show a security warning. You can proceed past this warning to access the webapp. +- **API Routes**: Verify that API endpoints like `https://cvmanager.local.com/user-auth` route to the CV Manager API. +- **Auth Routes**: Verify that Keycloak endpoints like `https://cvmanager.local.com/realms/cvmanager/.well-known/openid-configuration` route to the Keycloak instance. + +#### Important Note on Service Endpoints + +When using the NGINX proxy, all services are accessed through the same domain and port (443). Ensure your `.env` file reflects this. Specifically, the `KEYCLOAK_ENDPOINT` should point to the base domain: + +```env +KEYCLOAK_ENDPOINT=https://cvmanager.local.com +WEBAPP_ENDPOINT=https://cvmanager.local.com +API_ENDPOINT=https://cvmanager.local.com +``` + +#### Troubleshooting + +- **Certificate Errors**: If NGINX fails to start, check the logs (`docker compose logs cvmanager_nginx_proxy`) to ensure `server.crt` and `server.key` are correctly mounted in `/etc/ssl/certs/` and `/etc/ssl/private/` respectively. +- **Domain Resolution**: If you cannot reach the domain, ensure your `hosts` file was saved correctly and that you are not behind a VPN/proxy that interferes with local resolution. +- **Port Conflicts**: Ensure ports 80 and 443 are not being used by other services on your host machine. diff --git a/resources/nginx/gen_dhparam.sh b/resources/nginx/gen_dhparam.sh new file mode 100644 index 000000000..f3e402291 --- /dev/null +++ b/resources/nginx/gen_dhparam.sh @@ -0,0 +1,4 @@ +#!/bin/sh +if [ ! -f /etc/nginx/dhparam.pem ]; then + openssl dhparam -out /etc/nginx/dhparam.pem 2048 +fi \ No newline at end of file diff --git a/resources/nginx/nginx-ssl.conf b/resources/nginx/nginx-ssl.conf new file mode 100644 index 000000000..c0a262734 --- /dev/null +++ b/resources/nginx/nginx-ssl.conf @@ -0,0 +1,519 @@ +events { + worker_connections 768; +} + +http { + + upstream api { + server cvmanager_api:5000; + } + + upstream keycloak { + server cvmanager_keycloak:8080; + } + + upstream webapp { + server cvmanager_webapp:80; + } + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + ## + # SSL Settings + ## + + ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3; + ssl_prefer_server_ciphers on; + ssl_dhparam "/etc/nginx/dhparam.pem"; + add_header Strict-Transport-Security "max-age=31557600; includeSubDomains"; + ssl_stapling on; + ssl_stapling_verify on; + + ## + # General Logging Settings + ## + + access_log /var/log/nginx/access.log; + error_log /var/log/nginx/access.log; + + # HTTP forwards everything to HTTPS + # + server { + listen 80; + + server_name $SERVER_HOST; + + location / { + return 301 https://$host$request_uri; + } + + # deny access to .htaccess files, if Apache's document root + # concurs with nginx's one + location ~ /\.ht { + deny all; + } + } + + server { + listen 443 default_server ssl; + server_name $SERVER_HOST; + + ssl_certificate /etc/ssl/certs/server.crt; + ssl_certificate_key /etc/ssl/private/server.key; + + client_max_body_size 300M; + + location ~ /\.ht { + deny all; + } + + # webapp endpoints + location / { + proxy_pass http://webapp/; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/webapp-access.log; + } + + location /dashboard { + proxy_pass http://webapp/dashboard; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/webapp-access.log; + } + + # keycloak endpoints + location /admin { + proxy_pass http://keycloak/admin; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /realms { + proxy_pass http://keycloak/realms; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /resources { + proxy_pass http://keycloak/resources; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /user-auth { + proxy_pass http://api/user-auth; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /rsuinfo { + proxy_pass http://api/rsuinfo; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /rsu-online-status { + proxy_pass http://api/rsu-online-status; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /rsucounts { + proxy_pass http://api/rsucounts; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /rsu-msgfwd-query { + proxy_pass http://api/rsu-msgfwd-query; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /rsu-command { + proxy_pass http://api/rsu-command; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /rsu-config-geo-query { + proxy_pass http://api/rsu-config-geo-query; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /rsu-error-summary { + proxy_pass http://api/rsu-error-summary; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /wzdx-feed { + proxy_pass http://api/wzdx-feed; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /rsu-geo-msg-data { + proxy_pass http://api/rsu-geo-msg-data; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /iss-scms-status { + proxy_pass http://api/iss-scms-status; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /rsu-ssm-srm-data { + proxy_pass http://api/rsu-ssm-srm-data; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /admin-new-rsu { + proxy_pass http://api/admin-new-rsu; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /admin-rsu { + proxy_pass http://api/admin-rsu; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /admin-new-user { + proxy_pass http://api/admin-new-user; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /admin-user { + proxy_pass http://api/admin-user; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /admin-new-org { + proxy_pass http://api/admin-new-org; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /admin-org { + proxy_pass http://api/admin-org; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /contact-support { + proxy_pass http://api/contact-support; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /admin-new-notification { + proxy_pass http://api/admin-new-notification; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + location /admin-notification { + proxy_pass http://api/admin-notification; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Range $http_range; + + proxy_http_version 1.1; + proxy_set_header Connection ''; + proxy_buffering off; + proxy_cache off; + proxy_read_timeout 86400; + + access_log /var/log/nginx/access.log; + } + + } +} \ No newline at end of file diff --git a/sample.env b/sample.env index 1361ca207..3d314e173 100644 --- a/sample.env +++ b/sample.env @@ -5,21 +5,23 @@ # Additionally, each individual service in this project can be started by specifying its service name as a profile. # The currently available profile groups are listed below. # basic, webapp, mongo_full, kafka_full, intersection, intersection_no_api, conflictmonitor, addons, obu_ota, kafka_connect_standalone -COMPOSE_PROFILES=basic,webapp,mongo_full,kafka_full,intersection,kafka_connect_standalone +COMPOSE_PROFILES=nginx_proxy,basic,webapp,addons ######## -------- General Variables - Apply to All Profiles DOCKER_HOST_IP= -KEYCLOAK_DOMAIN=cvmanager.auth.com +KEYCLOAK_DOMAIN=cvmanager.local.com KC_HOST_IP=${DOCKER_HOST_IP} -KEYCLOAK_ENDPOINT=http://${KEYCLOAK_DOMAIN}:8084 - +KEYCLOAK_ENDPOINT=https://${KEYCLOAK_DOMAIN} WEBAPP_DOMAIN=cvmanager.local.com WEBAPP_HOST_IP=${DOCKER_HOST_IP} -WEBAPP_ENDPOINT=http://${WEBAPP_DOMAIN} +WEBAPP_ENDPOINT=https://${WEBAPP_DOMAIN} + +API_ENDPOINT=https://${WEBAPP_DOMAIN} -API_ENDPOINT=http://cvmanager.local.com:8081 +# NGINX Resources path +NGINX_PROXY_RESOURCES=./resources/nginx # Remote dockerhub variables for ODE and ConflictMonitor images DOCKERHUB_HOST=usdotjpoode @@ -80,6 +82,13 @@ PG_DB_NAME=postgres PG_DB_USER=postgres # If the PG_DB_PASS variable has special characters, make sure to wrap it in single quotes PG_DB_PASS=postgres +# Set to 'False' to disable SSL when connecting to the Postgres database +PG_SSL_REQUIRED=False + +# CA Certificate truststore path for the API service +# This is used by Python's requests library and Keycloak verification +REQUESTS_CA_BUNDLE=/etc/ssl/ca/ca.crt +KEYCLOAK_VERIFY=/etc/ssl/ca/ca.crt #### ---- cvmanager_api @@ -454,4 +463,4 @@ NGINX_ENCRYPTION="plain" # SSL file name in path /docker/nginx/ssl/ SERVER_CERT_FILE="ota_server.crt" -SERVER_KEY_FILE="ota_server.key" +SERVER_KEY_FILE="ota_server.key" \ No newline at end of file diff --git a/services/Dockerfile.api b/services/Dockerfile.api index 9237a9aaf..2a9647be1 100644 --- a/services/Dockerfile.api +++ b/services/Dockerfile.api @@ -1,8 +1,12 @@ FROM python:3.12.2-slim -# Prepare the SNMP functionality for the image -RUN apt-get update && apt-get install -y snmpd && apt-get install -y snmp -COPY resources/mibs/* /usr/share/snmp/mibs/ +# Prepare the SNMP and CA certificate functionality for the image +RUN apt-get update && \ + apt-get install -y snmpd snmp ca-certificates +COPY resources/mibs/ /usr/share/snmp/mibs/ + +COPY resources/ca/ca.crt /usr/local/share/ca-certificates/ca.crt +RUN update-ca-certificates # Allow statements and log messages to immediately appear in the Knative logs ENV PYTHONUNBUFFERED=True @@ -23,4 +27,4 @@ RUN pip install --no-cache-dir -r requirements.txt # For environments with multiple CPU cores, increase the number of workers # to be equal to the cores available. # Timeout is set to 0 to disable the timeouts of the workers to allow Cloud Run to handle instance scaling. -CMD exec gunicorn --bind :5000 --workers 1 --threads 8 --timeout 0 main:app \ No newline at end of file +CMD update-ca-certificates && exec gunicorn --bind :5000 --workers 1 --threads 8 --timeout 0 main:app \ No newline at end of file diff --git a/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py b/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py index 3b965c5c9..379bed5fc 100644 --- a/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py +++ b/services/addons/images/firmware_manager/upgrade_scheduler/upgrade_scheduler.py @@ -33,7 +33,7 @@ # Changed from a constant to a function to help with unit testing def get_upgrade_limit() -> int: try: - upgrade_limit = int(os.environ.get("ACTIVE_UPGRADE_LIMIT", "1")) + upgrade_limit = int(os.environ.get("ACTIVE_UPGRADE_LIMIT", "4")) return upgrade_limit except ValueError: raise ValueError( diff --git a/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py b/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py index 4a10f58f2..06e47882f 100644 --- a/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py +++ b/services/addons/tests/firmware_manager/upgrade_scheduler/test_upgrade_scheduler.py @@ -1101,7 +1101,7 @@ def test_init_background_task(mock_bgscheduler): def test_get_upgrade_limit_no_env(): limit = upgrade_scheduler.get_upgrade_limit() - assert limit == 1 + assert limit == 4 @patch.dict("os.environ", {"ACTIVE_UPGRADE_LIMIT": "5"}) diff --git a/services/api/src/middleware.py b/services/api/src/middleware.py index 0ca717659..41c81249b 100644 --- a/services/api/src/middleware.py +++ b/services/api/src/middleware.py @@ -38,11 +38,18 @@ class FEATURE_KEYS_LITERAL(Enum): def get_user_role(token) -> UserInfo | None: # TODO: Consider using pythjon-jose or PyJWT to locally validate the token, instead of calling the Keycloak server + keycloak_verify = os.getenv("KEYCLOAK_VERIFY", "True") + if keycloak_verify.lower() == "true": + keycloak_verify = True + elif keycloak_verify.lower() == "false": + keycloak_verify = False + keycloak_openid = KeycloakOpenID( server_url=os.getenv("KEYCLOAK_ENDPOINT"), realm_name=os.getenv("KEYCLOAK_REALM"), client_id=os.getenv("KEYCLOAK_API_CLIENT_ID"), client_secret_key=os.getenv("KEYCLOAK_API_CLIENT_SECRET_KEY"), + verify=keycloak_verify, ) introspect = keycloak_openid.introspect(token) data = None diff --git a/services/common/pgquery.py b/services/common/pgquery.py index 40c9e02f6..4c1675d8f 100644 --- a/services/common/pgquery.py +++ b/services/common/pgquery.py @@ -1,4 +1,5 @@ import os +import ssl import sqlalchemy import logging @@ -23,6 +24,25 @@ def init_tcp_connection_engine(db_user, db_pass, db_name, db_hostname, db_port): logging.info("Creating DB pool") logging.debug(f"{db_user},{db_name},{db_hostname},{db_port}") + + pg_ssl_required = os.getenv("PG_SSL_REQUIRED", "True").lower() == "true" + connect_args = {} + + if pg_ssl_required: + ssl_context = ssl.create_default_context() + ca_cert_path = "/etc/ssl/ca/ca.crt" + if os.path.exists(ca_cert_path): + ssl_context.load_verify_locations(ca_cert_path) + else: + logging.warning( + f"CA certificate not found at {ca_cert_path}. Skipping SSL verification for Postgres." + ) + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE + connect_args["ssl_context"] = ssl_context + else: + logging.info("PG_SSL_REQUIRED is False. Connecting to Postgres without SSL.") + pool = sqlalchemy.create_engine( # Equivalent URL: # postgresql+pg8000://:@:/ @@ -34,6 +54,7 @@ def init_tcp_connection_engine(db_user, db_pass, db_name, db_hostname, db_port): port=db_port, # e.g. 5432 database=db_name, # e.g. "my-database-name" ), + connect_args=connect_args, **db_config, ) # pool.dialect.description_encoding = None diff --git a/services/common/tests/test_pgquery.py b/services/common/tests/test_pgquery.py index ab3c40f05..8c98dc204 100644 --- a/services/common/tests/test_pgquery.py +++ b/services/common/tests/test_pgquery.py @@ -5,6 +5,7 @@ # test that init_tcp_connection_engine is calling sqlalchemy.create_engine with expected arguments +@patch.dict(os.environ, {"PG_SSL_REQUIRED": "False"}) @patch( "common.pgquery.db_config", new={"pool_size": 5, "max_overflow": 2, "pool_timeout": 30, "pool_recycle": 1800}, @@ -44,8 +45,9 @@ def test_init_tcp_connection_engine(): "pool_timeout": 30, "pool_recycle": 1800, } - sqlalchemy.create_engine.assert_called_once_with("my_url", **my_db_config) - + sqlalchemy.create_engine.assert_called_once_with( + "my_url", connect_args={}, **my_db_config + ) # test that init_socket_connection_engine is calling sqlalchemy.create_engine with expected arguments @patch(