From 8cb0579d34900bf900e4bce8923f60146e4e38c4 Mon Sep 17 00:00:00 2001 From: Mark Vrijlandt Date: Tue, 7 Apr 2026 20:46:39 +0200 Subject: [PATCH 1/2] use postgres for time series storage --- .env.template | 4 ++ .vscode/launch.json | 17 ++++++ docker-compose.override.dev.yml | 22 ++++---- docker-compose.override.setup.yml | 2 +- docker-compose.yml | 19 +++---- example_sdk_client/optimizer.py | 14 ++--- postgres/postgres-init.sql | 69 +++++++++++++++++++++--- scripts/setup.ps1 | 26 ++++----- scripts/setup.sh | 16 +++--- scripts/start-dev.sh | 2 +- scripts/start.sh | 2 +- scripts/start_postgres_in_dev_mode.sh | 4 +- system_tests/docker-compose.override.yml | 4 +- track_memory.sh | 4 ++ 14 files changed, 142 insertions(+), 63 deletions(-) create mode 100644 .vscode/launch.json create mode 100755 track_memory.sh diff --git a/.env.template b/.env.template index 8478243..3659dc5 100644 --- a/.env.template +++ b/.env.template @@ -3,6 +3,10 @@ POSTGRES_ROOT_PASSWORD=1234 POSTGRES_DEV_PORT=6432 POSTGRES_ORCHESTRATOR_USER_NAME=omotes_orchestrator POSTGRES_ORCHESTRATOR_USER_PASSWORD=somepass3 +POSTGRES_TIMESERIES_RW_USER_NAME=omotes_timeseries_rw +POSTGRES_TIMESERIES_RW_USER_PASSWORD=somepass4 +POSTGRES_TIMESERIES_RO_USER_NAME=omotes_timeseries_ro +POSTGRES_TIMESERIES_RO_USER_PASSWORD=somepass5 RABBITMQ_PORT=5672 RABBITMQ_MANAGEMENT_PORT=15672 diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..92f667f --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,17 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Run optimizer.py", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/example_sdk_client/optimizer.py", + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/example_sdk_client", + "justMyCode": true, + "env": { + "PYTHONPATH": "${workspaceFolder}/example_sdk_client" + } + } + ] +} \ No newline at end of file diff --git a/docker-compose.override.dev.yml b/docker-compose.override.dev.yml index 301455f..0bc04ad 100644 --- a/docker-compose.override.dev.yml +++ b/docker-compose.override.dev.yml @@ -1,10 +1,10 @@ services: - orchestrator: - image: omotes-orchestrator-dev - build: - context: .. - dockerfile: orchestrator/dev.Dockerfile +# orchestrator: +# image: omotes-orchestrator-dev +# build: +# context: .. +# dockerfile: orchestrator/dev.Dockerfile grow_worker_optimizer: &grow_worker image: omotes-worker-mesido-dev @@ -27,12 +27,12 @@ services: grow_worker_gurobi: <<: *grow_worker - omotes_simulator_worker: - image: omotes-worker-simulator-dev - build: - context: .. - dockerfile: simulator-worker/dev.Dockerfile +# omotes_simulator_worker: +# image: omotes-worker-simulator-dev +# build: +# context: .. +# dockerfile: simulator-worker/dev.Dockerfile - orchestrator_postgres_db: + omotes_postgres_db: ports: - "${POSTGRES_DEV_PORT}:5432" diff --git a/docker-compose.override.setup.yml b/docker-compose.override.setup.yml index 2d2e023..03f2702 100644 --- a/docker-compose.override.setup.yml +++ b/docker-compose.override.setup.yml @@ -18,6 +18,6 @@ services: INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD: ${INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD} INFLUXDB_PORT: ${INFLUXDB_PORT} - orchestrator_postgres_db: + omotes_postgres_db: volumes: - "${POSTGRES_INIT_FILE:-./postgres/postgres-init.sql}:/setup/init.sql:ro" diff --git a/docker-compose.yml b/docker-compose.yml index df64bb4..511641a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,7 +60,7 @@ services: - ./influxdb/influxdb.conf:/etc/influxdb/influxdb.conf:ro - influxdb_storage:/var/lib/influxdb - orchestrator_postgres_db: &orchestrator_postgres_db + omotes_postgres_db: &omotes_postgres_db image: postgres:15.4 restart: unless-stopped volumes: @@ -69,7 +69,6 @@ services: - omotes environment: PGDATA: /var/lib/postgresql/data - POSTGRES_DB: omotes_jobs POSTGRES_USER: ${POSTGRES_ROOT_USER} POSTGRES_PASSWORD: ${POSTGRES_ROOT_PASSWORD} @@ -80,18 +79,16 @@ services: "CMD", "pg_isready", "-U", - "${POSTGRES_ROOT_USER}", - "-d", - "omotes_jobs", + "${POSTGRES_ROOT_USER}" ] interval: 5s timeout: 5s retries: 10 # Skipped in 'docker-compose up', only used to develop database revisions. - orchestrator_postgres_db_dev: - <<: *orchestrator_postgres_db - hostname: orchestrator_postgres_db + omotes_postgres_db_dev: + <<: *omotes_postgres_db + hostname: omotes_postgres_db ports: - "${POSTGRES_DEV_PORT}:5432" profiles: @@ -123,7 +120,7 @@ services: TASK_RABBITMQ_PORT: 5672 TASK_RABBITMQ_VIRTUALHOST: omotes_celery - POSTGRESQL_HOST: orchestrator_postgres_db + POSTGRESQL_HOST: omotes_postgres_db POSTGRESQL_PORT: 5432 POSTGRESQL_DATABASE: omotes_jobs POSTGRESQL_USERNAME: ${POSTGRES_ORCHESTRATOR_USER_NAME} @@ -143,13 +140,13 @@ services: depends_on: rabbitmq: condition: service_healthy - orchestrator_postgres_db: + omotes_postgres_db: condition: service_healthy omotes_influxdb: condition: service_healthy grow_worker_optimizer: &grow_worker - image: ghcr.io/project-omotes/omotes-grow-optimizer-worker:2.1.3 + image: ghcr.io/project-omotes/omotes-grow-optimizer-worker:2.1.7 restart: unless-stopped deploy: &grow_worker_deploy replicas: 2 diff --git a/example_sdk_client/optimizer.py b/example_sdk_client/optimizer.py index b3f692a..134679e 100644 --- a/example_sdk_client/optimizer.py +++ b/example_sdk_client/optimizer.py @@ -11,17 +11,17 @@ JobStatusUpdate, ) -rabbitmq_config = RabbitMQConfig(username="omotes", password="somepass1", virtual_host="omotes") +rabbitmq_config = RabbitMQConfig(port=5673,username="omotes", password="somepass1", virtual_host="omotes") STOP_EVENT = threading.Event() def handle_on_finished(job: Job, result: JobResult): - print("--------------Logs:") - print(result.logs) - print() - print("--------------ESDL:") - print(result.output_esdl) - print() + # print("--------------Logs:") + # print(result.logs) + # print() + # print("--------------ESDL:") + # print(result.output_esdl) + # print() print("--------------ESDL messages:") print(result.esdl_messages) print("--------------Result") diff --git a/postgres/postgres-init.sql b/postgres/postgres-init.sql index dc3ebbb..f80fa34 100644 --- a/postgres/postgres-init.sql +++ b/postgres/postgres-init.sql @@ -1,8 +1,61 @@ -CREATE USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD'; -ALTER USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD'; -GRANT ALL PRIVILEGES ON DATABASE :PG_DB TO :PG_USERNAME; -GRANT ALL PRIVILEGES ON SCHEMA public TO :PG_USERNAME; -GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO :PG_USERNAME; -GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO :PG_USERNAME; -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :PG_USERNAME; -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :PG_USERNAME; +-- CREATE USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD'; +-- ALTER USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD'; +-- GRANT ALL PRIVILEGES ON DATABASE :PG_DB TO :PG_USERNAME; +-- GRANT ALL PRIVILEGES ON SCHEMA public TO :PG_USERNAME; +-- GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO :PG_USERNAME; +-- GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO :PG_USERNAME; +-- ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :PG_USERNAME; +-- ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :PG_USERNAME; + + +-- create databases +CREATE DATABASE :PG_DB_JOBS; +CREATE DATABASE :PG_DB_TIMESERIES; + +-- create roles +CREATE ROLE :JOBS_RW_USER_NAME LOGIN PASSWORD :'JOBS_RW_USER_PASSWORD'; +CREATE ROLE :TIMESERIES_RW_USER_NAME LOGIN PASSWORD :'TIMESERIES_RW_USER_PASSWORD'; +CREATE ROLE :TIMESERIES_RO_USER_NAME LOGIN PASSWORD :'TIMESERIES_RO_USER_PASSWORD'; + +-- DB1 +\connect :PG_DB_JOBS + +-- Make RW user owner +ALTER DATABASE :PG_DB_JOBS OWNER TO :JOBS_RW_USER_NAME; +ALTER SCHEMA public OWNER TO :JOBS_RW_USER_NAME; + +-- Ensure RW can create objects +GRANT ALL ON SCHEMA public TO :JOBS_RW_USER_NAME; + +-- Existing objects +GRANT ALL ON ALL TABLES IN SCHEMA public TO :JOBS_RW_USER_NAME; +GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO :JOBS_RW_USER_NAME; +GRANT ALL ON ALL FUNCTIONS IN SCHEMA public TO :JOBS_RW_USER_NAME; + +-- Default privileges +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :JOBS_RW_USER_NAME; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :JOBS_RW_USER_NAME; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON FUNCTIONS TO :JOBS_RW_USER_NAME; + +-- DB2 +\connect :PG_DB_TIMESERIES + +-- Make RW owner +ALTER DATABASE :PG_DB_TIMESERIES OWNER TO :TIMESERIES_RW_USER_NAME; +ALTER SCHEMA public OWNER TO :TIMESERIES_RW_USER_NAME; + +-- Schema privileges +GRANT ALL ON SCHEMA public TO :TIMESERIES_RW_USER_NAME; +GRANT USAGE ON SCHEMA public TO :TIMESERIES_RO_USER_NAME; + +-- Existing objects +GRANT ALL ON ALL TABLES IN SCHEMA public TO :TIMESERIES_RW_USER_NAME; +GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO :TIMESERIES_RW_USER_NAME; +GRANT ALL ON ALL FUNCTIONS IN SCHEMA public TO :TIMESERIES_RW_USER_NAME; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO :TIMESERIES_RO_USER_NAME; + +-- Default privileges +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :TIMESERIES_RW_USER_NAME; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :TIMESERIES_RW_USER_NAME; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON FUNCTIONS TO :TIMESERIES_RW_USER_NAME; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO :TIMESERIES_RO_USER_NAME; diff --git a/scripts/setup.ps1 b/scripts/setup.ps1 index 023f568..21776fd 100644 --- a/scripts/setup.ps1 +++ b/scripts/setup.ps1 @@ -13,21 +13,21 @@ get-content .env | foreach { # docker compose stop # Deploy postgres omotes schema -docker compose --profile=manual_dev down orchestrator_postgres_db_dev -docker compose up -d --wait orchestrator_postgres_db -docker compose exec orchestrator_postgres_db psql -d postgres -c 'CREATE DATABASE omotes_jobs;' -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "CREATE USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "ALTER USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON DATABASE omotes_jobs TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose --profile=manual_dev down omotes_postgres_db_dev +docker compose up -d --wait postgres_db +docker compose exec omotes_postgres_db psql -d postgres -c 'CREATE DATABASE omotes_jobs;' +docker compose exec omotes_postgres_db psql -d omotes_jobs -c "CREATE USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" +docker compose exec omotes_postgres_db psql -d omotes_jobs -c "ALTER USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" +docker compose exec omotes_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON DATABASE omotes_jobs TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" # Upgrade omotes tables -docker compose build orchestrator_postgres_db_upgrade -docker compose run --rm orchestrator_postgres_db_upgrade +docker compose build omotes_postgres_db_upgrade +docker compose run --rm omotes_postgres_db_upgrade # Setup rabbitmq docker compose up -d --wait rabbitmq diff --git a/scripts/setup.sh b/scripts/setup.sh index 57ca0cf..cd86471 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -13,12 +13,16 @@ echo "Using docker compose file at: $DOCKER_COMPOSE_FILE" $DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE --profile=manual_dev down # add postgres user with privileges -$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE up --wait orchestrator_postgres_db -$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE exec orchestrator_postgres_db psql \ - -d omotes_jobs \ - -v PG_USERNAME="$POSTGRES_ORCHESTRATOR_USER_NAME" \ - -v PG_PASSWORD="$POSTGRES_ORCHESTRATOR_USER_PASSWORD" \ - -v PG_DB=omotes_jobs \ +$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE up --wait omotes_postgres_db +$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE exec omotes_postgres_db psql \ + -v PG_DB_JOBS=omotes_jobs \ + -v JOBS_RW_USER_NAME="$POSTGRES_ORCHESTRATOR_USER_NAME" \ + -v JOBS_RW_USER_PASSWORD="$POSTGRES_ORCHESTRATOR_USER_PASSWORD" \ + -v PG_DB_TIMESERIES=omotes_timeseries \ + -v TIMESERIES_RW_USER_NAME="$POSTGRES_TIMESERIES_RW_USER_NAME" \ + -v TIMESERIES_RW_USER_PASSWORD="$POSTGRES_TIMESERIES_RW_USER_PASSWORD" \ + -v TIMESERIES_RO_USER_NAME="$POSTGRES_TIMESERIES_RO_USER_NAME" \ + -v TIMESERIES_RO_USER_PASSWORD="$POSTGRES_TIMESERIES_RO_USER_PASSWORD" \ -f /setup/init.sql # add rabbitmq 'omotes' and 'celery' vhosts and users diff --git a/scripts/start-dev.sh b/scripts/start-dev.sh index 8966e7c..ce10249 100755 --- a/scripts/start-dev.sh +++ b/scripts/start-dev.sh @@ -2,6 +2,6 @@ . scripts/_select_docker_compose.sh -$DOCKER_COMPOSE --profile=manual_dev down orchestrator_postgres_db_dev +$DOCKER_COMPOSE --profile=manual_dev down omotes_postgres_db_dev $DOCKER_COMPOSE -f docker-compose.yml -f docker-compose.override.dev.yml build $DOCKER_COMPOSE -f docker-compose.yml -f docker-compose.override.dev.yml up --wait diff --git a/scripts/start.sh b/scripts/start.sh index 8592058..a3ef662 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -2,5 +2,5 @@ . scripts/_select_docker_compose.sh -$DOCKER_COMPOSE --profile=manual_dev down orchestrator_postgres_db_dev +$DOCKER_COMPOSE --profile=manual_dev down omotes_postgres_db_dev $DOCKER_COMPOSE up --wait diff --git a/scripts/start_postgres_in_dev_mode.sh b/scripts/start_postgres_in_dev_mode.sh index d899b06..13da87f 100755 --- a/scripts/start_postgres_in_dev_mode.sh +++ b/scripts/start_postgres_in_dev_mode.sh @@ -1,4 +1,4 @@ #!/bin/bash -docker compose -f docker-compose.yml down orchestrator_postgres_db -docker compose -f docker-compose.yml up --wait orchestrator_postgres_db_dev +docker compose -f docker-compose.yml down omotes_postgres_db +docker compose -f docker-compose.yml up --wait omotes_postgres_db_dev diff --git a/system_tests/docker-compose.override.yml b/system_tests/docker-compose.override.yml index ba603c6..3f6f8a4 100644 --- a/system_tests/docker-compose.override.yml +++ b/system_tests/docker-compose.override.yml @@ -17,7 +17,7 @@ services: RABBITMQ_HOST: rabbitmq RABBITMQ_PORT: 5672 - POSTGRES_HOST: orchestrator_postgres_db + POSTGRES_HOST: omotes_postgres_db POSTGRES_PORT: 5432 POSTGRES_ORCHESTRATOR_USER_NAME: omotes_orchestrator POSTGRES_ORCHESTRATOR_USER_PASSWORD: somepass3 @@ -32,6 +32,6 @@ services: TIMEOUT_JOB_HANDLER_RERUN_SEC: 5 JOB_RETENTION_SEC: 20 - orchestrator_postgres_db: + omotes_postgres_db: ports: - "${POSTGRES_DEV_PORT}:5432" diff --git a/track_memory.sh b/track_memory.sh new file mode 100755 index 0000000..46fdc1d --- /dev/null +++ b/track_memory.sh @@ -0,0 +1,4 @@ +while true; do + date +"%Y-%m-%d %H:%M:%S" + docker stats 79e4e234711e --no-stream --format "{{.Name}} {{.MemUsage}}" +done >> container_mem.log \ No newline at end of file From da3e913868bd49d2adad5e8985175aa5ced01f59 Mon Sep 17 00:00:00 2001 From: Mark Vrijlandt Date: Fri, 5 Jun 2026 11:47:18 +0200 Subject: [PATCH 2/2] timeseries in postgres enabled --- .env.template | 2 +- .vscode/launch.json | 18 ++++++++++++++++ docker-compose.override.dev.yml | 4 ---- docker-compose.override.setup.yml | 2 +- docker-compose.yml | 26 +++++++++++------------- postgres/postgres-init.sql | 1 + scripts/setup.ps1 | 24 +++++++++++----------- scripts/setup.sh | 9 +++++--- scripts/start-dev.sh | 2 +- scripts/start.sh | 2 +- scripts/start_postgres_in_dev_mode.sh | 4 ---- scripts/stop.sh | 2 +- system_tests/docker-compose.override.yml | 4 ++-- 13 files changed, 56 insertions(+), 44 deletions(-) delete mode 100755 scripts/start_postgres_in_dev_mode.sh diff --git a/.env.template b/.env.template index 3659dc5..706df58 100644 --- a/.env.template +++ b/.env.template @@ -1,6 +1,6 @@ POSTGRES_ROOT_USER=root POSTGRES_ROOT_PASSWORD=1234 -POSTGRES_DEV_PORT=6432 +POSTGRES_PORT=6432 POSTGRES_ORCHESTRATOR_USER_NAME=omotes_orchestrator POSTGRES_ORCHESTRATOR_USER_PASSWORD=somepass3 POSTGRES_TIMESERIES_RW_USER_NAME=omotes_timeseries_rw diff --git a/.vscode/launch.json b/.vscode/launch.json index 92f667f..792f947 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -12,6 +12,24 @@ "env": { "PYTHONPATH": "${workspaceFolder}/example_sdk_client" } + }, + { + "name": "test__simulator__multiple_ates_run", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "args": [ + "test_workflows_steps.py::TestWorkflows::test__simulator__multiple_ates_run", + "-v", + "-s" + ], + "jinja": true, + "justMyCode": true, + "cwd": "${workspaceFolder}/system_tests/src", + "envFile": "${workspaceFolder}/.env.test", + "env": { + "PYTHONPATH": "${workspaceFolder}/system_tests" + } } ] } \ No newline at end of file diff --git a/docker-compose.override.dev.yml b/docker-compose.override.dev.yml index 0bc04ad..4a6b573 100644 --- a/docker-compose.override.dev.yml +++ b/docker-compose.override.dev.yml @@ -32,7 +32,3 @@ services: # build: # context: .. # dockerfile: simulator-worker/dev.Dockerfile - - omotes_postgres_db: - ports: - - "${POSTGRES_DEV_PORT}:5432" diff --git a/docker-compose.override.setup.yml b/docker-compose.override.setup.yml index 03f2702..8b85c7f 100644 --- a/docker-compose.override.setup.yml +++ b/docker-compose.override.setup.yml @@ -18,6 +18,6 @@ services: INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD: ${INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD} INFLUXDB_PORT: ${INFLUXDB_PORT} - omotes_postgres_db: + omotes_postgres: volumes: - "${POSTGRES_INIT_FILE:-./postgres/postgres-init.sql}:/setup/init.sql:ro" diff --git a/docker-compose.yml b/docker-compose.yml index 511641a..f613d7d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,11 +60,16 @@ services: - ./influxdb/influxdb.conf:/etc/influxdb/influxdb.conf:ro - influxdb_storage:/var/lib/influxdb - omotes_postgres_db: &omotes_postgres_db + omotes_postgres: &omotes_postgres image: postgres:15.4 restart: unless-stopped + command: ["postgres", "-p", "${POSTGRES_PORT}"] volumes: - "db-data:/var/lib/postgresql/data/" + expose: + - "${POSTGRES_PORT}" + ports: + - "${POSTGRES_PORT}:${POSTGRES_PORT}" networks: - omotes environment: @@ -79,21 +84,14 @@ services: "CMD", "pg_isready", "-U", - "${POSTGRES_ROOT_USER}" + "${POSTGRES_ROOT_USER}", + "-p", + "${POSTGRES_PORT}" ] interval: 5s timeout: 5s retries: 10 - # Skipped in 'docker-compose up', only used to develop database revisions. - omotes_postgres_db_dev: - <<: *omotes_postgres_db - hostname: omotes_postgres_db - ports: - - "${POSTGRES_DEV_PORT}:5432" - profiles: - - manual_dev - orchestrator: image: ghcr.io/project-omotes/omotes_orchestrator:1.2.6 restart: unless-stopped @@ -120,8 +118,8 @@ services: TASK_RABBITMQ_PORT: 5672 TASK_RABBITMQ_VIRTUALHOST: omotes_celery - POSTGRESQL_HOST: omotes_postgres_db - POSTGRESQL_PORT: 5432 + POSTGRESQL_HOST: omotes_postgres + POSTGRESQL_PORT: ${POSTGRES_PORT} POSTGRESQL_DATABASE: omotes_jobs POSTGRESQL_USERNAME: ${POSTGRES_ORCHESTRATOR_USER_NAME} POSTGRESQL_PASSWORD: ${POSTGRES_ORCHESTRATOR_USER_PASSWORD} @@ -140,7 +138,7 @@ services: depends_on: rabbitmq: condition: service_healthy - omotes_postgres_db: + omotes_postgres: condition: service_healthy omotes_influxdb: condition: service_healthy diff --git a/postgres/postgres-init.sql b/postgres/postgres-init.sql index f80fa34..010e4c0 100644 --- a/postgres/postgres-init.sql +++ b/postgres/postgres-init.sql @@ -15,6 +15,7 @@ CREATE DATABASE :PG_DB_TIMESERIES; -- create roles CREATE ROLE :JOBS_RW_USER_NAME LOGIN PASSWORD :'JOBS_RW_USER_PASSWORD'; CREATE ROLE :TIMESERIES_RW_USER_NAME LOGIN PASSWORD :'TIMESERIES_RW_USER_PASSWORD'; +ALTER ROLE :TIMESERIES_RW_USER_NAME CREATEDB; CREATE ROLE :TIMESERIES_RO_USER_NAME LOGIN PASSWORD :'TIMESERIES_RO_USER_PASSWORD'; -- DB1 diff --git a/scripts/setup.ps1 b/scripts/setup.ps1 index 21776fd..e2e4e62 100644 --- a/scripts/setup.ps1 +++ b/scripts/setup.ps1 @@ -13,21 +13,21 @@ get-content .env | foreach { # docker compose stop # Deploy postgres omotes schema -docker compose --profile=manual_dev down omotes_postgres_db_dev +docker compose down docker compose up -d --wait postgres_db -docker compose exec omotes_postgres_db psql -d postgres -c 'CREATE DATABASE omotes_jobs;' -docker compose exec omotes_postgres_db psql -d omotes_jobs -c "CREATE USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" -docker compose exec omotes_postgres_db psql -d omotes_jobs -c "ALTER USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" -docker compose exec omotes_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON DATABASE omotes_jobs TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec omotes_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec omotes_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec omotes_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec omotes_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec omotes_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres psql -d postgres -c 'CREATE DATABASE omotes_jobs;' +docker compose exec omotes_postgres psql -d omotes_jobs -c "CREATE USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" +docker compose exec omotes_postgres psql -d omotes_jobs -c "ALTER USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" +docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON DATABASE omotes_jobs TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" # Upgrade omotes tables -docker compose build omotes_postgres_db_upgrade -docker compose run --rm omotes_postgres_db_upgrade +docker compose build omotes_postgres_upgrade +docker compose run --rm omotes_postgres_upgrade # Setup rabbitmq docker compose up -d --wait rabbitmq diff --git a/scripts/setup.sh b/scripts/setup.sh index cd86471..a6c90f0 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -10,11 +10,14 @@ DOCKER_COMPOSE_FILE=${2:-"./docker-compose.yml -f ./docker-compose.override.setu echo "Using docker compose file at: $DOCKER_COMPOSE_FILE" # stop system before setup -$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE --profile=manual_dev down +$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE down # add postgres user with privileges -$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE up --wait omotes_postgres_db -$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE exec omotes_postgres_db psql \ +$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE up --wait omotes_postgres +$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE exec omotes_postgres psql \ + -U "$POSTGRES_ROOT_USER" \ + -p "$POSTGRES_PORT" \ + -d postgres \ -v PG_DB_JOBS=omotes_jobs \ -v JOBS_RW_USER_NAME="$POSTGRES_ORCHESTRATOR_USER_NAME" \ -v JOBS_RW_USER_PASSWORD="$POSTGRES_ORCHESTRATOR_USER_PASSWORD" \ diff --git a/scripts/start-dev.sh b/scripts/start-dev.sh index ce10249..9f7c31d 100755 --- a/scripts/start-dev.sh +++ b/scripts/start-dev.sh @@ -2,6 +2,6 @@ . scripts/_select_docker_compose.sh -$DOCKER_COMPOSE --profile=manual_dev down omotes_postgres_db_dev +$DOCKER_COMPOSE down $DOCKER_COMPOSE -f docker-compose.yml -f docker-compose.override.dev.yml build $DOCKER_COMPOSE -f docker-compose.yml -f docker-compose.override.dev.yml up --wait diff --git a/scripts/start.sh b/scripts/start.sh index a3ef662..d963b44 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -2,5 +2,5 @@ . scripts/_select_docker_compose.sh -$DOCKER_COMPOSE --profile=manual_dev down omotes_postgres_db_dev +$DOCKER_COMPOSE down $DOCKER_COMPOSE up --wait diff --git a/scripts/start_postgres_in_dev_mode.sh b/scripts/start_postgres_in_dev_mode.sh deleted file mode 100755 index 13da87f..0000000 --- a/scripts/start_postgres_in_dev_mode.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -docker compose -f docker-compose.yml down omotes_postgres_db -docker compose -f docker-compose.yml up --wait omotes_postgres_db_dev diff --git a/scripts/stop.sh b/scripts/stop.sh index b23ebf4..cfa898d 100755 --- a/scripts/stop.sh +++ b/scripts/stop.sh @@ -1,4 +1,4 @@ #!/bin/bash . scripts/_select_docker_compose.sh -$DOCKER_COMPOSE --profile=manual_dev down +$DOCKER_COMPOSE down diff --git a/system_tests/docker-compose.override.yml b/system_tests/docker-compose.override.yml index 3f6f8a4..0cd1a94 100644 --- a/system_tests/docker-compose.override.yml +++ b/system_tests/docker-compose.override.yml @@ -17,7 +17,7 @@ services: RABBITMQ_HOST: rabbitmq RABBITMQ_PORT: 5672 - POSTGRES_HOST: omotes_postgres_db + POSTGRES_HOST: omotes_postgres POSTGRES_PORT: 5432 POSTGRES_ORCHESTRATOR_USER_NAME: omotes_orchestrator POSTGRES_ORCHESTRATOR_USER_PASSWORD: somepass3 @@ -32,6 +32,6 @@ services: TIMEOUT_JOB_HANDLER_RERUN_SEC: 5 JOB_RETENTION_SEC: 20 - omotes_postgres_db: + omotes_postgres: ports: - "${POSTGRES_DEV_PORT}:5432"