diff --git a/.env.template b/.env.template index 8478243..706df58 100644 --- a/.env.template +++ b/.env.template @@ -1,8 +1,12 @@ POSTGRES_ROOT_USER=root POSTGRES_ROOT_PASSWORD=1234 -POSTGRES_DEV_PORT=6432 +POSTGRES_PORT=6432 POSTGRES_ORCHESTRATOR_USER_NAME=omotes_orchestrator POSTGRES_ORCHESTRATOR_USER_PASSWORD=somepass3 +POSTGRES_TIMESERIES_RW_USER_NAME=omotes_timeseries_rw +POSTGRES_TIMESERIES_RW_USER_PASSWORD=somepass4 +POSTGRES_TIMESERIES_RO_USER_NAME=omotes_timeseries_ro +POSTGRES_TIMESERIES_RO_USER_PASSWORD=somepass5 RABBITMQ_PORT=5672 RABBITMQ_MANAGEMENT_PORT=15672 diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..792f947 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,35 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Run optimizer.py", + "type": "debugpy", + "request": "launch", + "program": "${workspaceFolder}/example_sdk_client/optimizer.py", + "console": "integratedTerminal", + "cwd": "${workspaceFolder}/example_sdk_client", + "justMyCode": true, + "env": { + "PYTHONPATH": "${workspaceFolder}/example_sdk_client" + } + }, + { + "name": "test__simulator__multiple_ates_run", + "type": "debugpy", + "request": "launch", + "module": "pytest", + "args": [ + "test_workflows_steps.py::TestWorkflows::test__simulator__multiple_ates_run", + "-v", + "-s" + ], + "jinja": true, + "justMyCode": true, + "cwd": "${workspaceFolder}/system_tests/src", + "envFile": "${workspaceFolder}/.env.test", + "env": { + "PYTHONPATH": "${workspaceFolder}/system_tests" + } + } + ] +} \ No newline at end of file diff --git a/docker-compose.override.dev.yml b/docker-compose.override.dev.yml index 301455f..4a6b573 100644 --- a/docker-compose.override.dev.yml +++ b/docker-compose.override.dev.yml @@ -1,10 +1,10 @@ services: - orchestrator: - image: omotes-orchestrator-dev - build: - context: .. - dockerfile: orchestrator/dev.Dockerfile +# orchestrator: +# image: omotes-orchestrator-dev +# build: +# context: .. +# dockerfile: orchestrator/dev.Dockerfile grow_worker_optimizer: &grow_worker image: omotes-worker-mesido-dev @@ -27,12 +27,8 @@ services: grow_worker_gurobi: <<: *grow_worker - omotes_simulator_worker: - image: omotes-worker-simulator-dev - build: - context: .. - dockerfile: simulator-worker/dev.Dockerfile - - orchestrator_postgres_db: - ports: - - "${POSTGRES_DEV_PORT}:5432" +# omotes_simulator_worker: +# image: omotes-worker-simulator-dev +# build: +# context: .. +# dockerfile: simulator-worker/dev.Dockerfile diff --git a/docker-compose.override.setup.yml b/docker-compose.override.setup.yml index 2d2e023..8b85c7f 100644 --- a/docker-compose.override.setup.yml +++ b/docker-compose.override.setup.yml @@ -18,6 +18,6 @@ services: INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD: ${INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD} INFLUXDB_PORT: ${INFLUXDB_PORT} - orchestrator_postgres_db: + omotes_postgres: volumes: - "${POSTGRES_INIT_FILE:-./postgres/postgres-init.sql}:/setup/init.sql:ro" diff --git a/docker-compose.yml b/docker-compose.yml index df64bb4..f613d7d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,16 +60,20 @@ services: - ./influxdb/influxdb.conf:/etc/influxdb/influxdb.conf:ro - influxdb_storage:/var/lib/influxdb - orchestrator_postgres_db: &orchestrator_postgres_db + omotes_postgres: &omotes_postgres image: postgres:15.4 restart: unless-stopped + command: ["postgres", "-p", "${POSTGRES_PORT}"] volumes: - "db-data:/var/lib/postgresql/data/" + expose: + - "${POSTGRES_PORT}" + ports: + - "${POSTGRES_PORT}:${POSTGRES_PORT}" networks: - omotes environment: PGDATA: /var/lib/postgresql/data - POSTGRES_DB: omotes_jobs POSTGRES_USER: ${POSTGRES_ROOT_USER} POSTGRES_PASSWORD: ${POSTGRES_ROOT_PASSWORD} @@ -81,22 +85,13 @@ services: "pg_isready", "-U", "${POSTGRES_ROOT_USER}", - "-d", - "omotes_jobs", + "-p", + "${POSTGRES_PORT}" ] interval: 5s timeout: 5s retries: 10 - # Skipped in 'docker-compose up', only used to develop database revisions. - orchestrator_postgres_db_dev: - <<: *orchestrator_postgres_db - hostname: orchestrator_postgres_db - ports: - - "${POSTGRES_DEV_PORT}:5432" - profiles: - - manual_dev - orchestrator: image: ghcr.io/project-omotes/omotes_orchestrator:1.2.6 restart: unless-stopped @@ -123,8 +118,8 @@ services: TASK_RABBITMQ_PORT: 5672 TASK_RABBITMQ_VIRTUALHOST: omotes_celery - POSTGRESQL_HOST: orchestrator_postgres_db - POSTGRESQL_PORT: 5432 + POSTGRESQL_HOST: omotes_postgres + POSTGRESQL_PORT: ${POSTGRES_PORT} POSTGRESQL_DATABASE: omotes_jobs POSTGRESQL_USERNAME: ${POSTGRES_ORCHESTRATOR_USER_NAME} POSTGRESQL_PASSWORD: ${POSTGRES_ORCHESTRATOR_USER_PASSWORD} @@ -143,13 +138,13 @@ services: depends_on: rabbitmq: condition: service_healthy - orchestrator_postgres_db: + omotes_postgres: condition: service_healthy omotes_influxdb: condition: service_healthy grow_worker_optimizer: &grow_worker - image: ghcr.io/project-omotes/omotes-grow-optimizer-worker:2.1.3 + image: ghcr.io/project-omotes/omotes-grow-optimizer-worker:2.1.7 restart: unless-stopped deploy: &grow_worker_deploy replicas: 2 diff --git a/example_sdk_client/optimizer.py b/example_sdk_client/optimizer.py index b3f692a..134679e 100644 --- a/example_sdk_client/optimizer.py +++ b/example_sdk_client/optimizer.py @@ -11,17 +11,17 @@ JobStatusUpdate, ) -rabbitmq_config = RabbitMQConfig(username="omotes", password="somepass1", virtual_host="omotes") +rabbitmq_config = RabbitMQConfig(port=5673,username="omotes", password="somepass1", virtual_host="omotes") STOP_EVENT = threading.Event() def handle_on_finished(job: Job, result: JobResult): - print("--------------Logs:") - print(result.logs) - print() - print("--------------ESDL:") - print(result.output_esdl) - print() + # print("--------------Logs:") + # print(result.logs) + # print() + # print("--------------ESDL:") + # print(result.output_esdl) + # print() print("--------------ESDL messages:") print(result.esdl_messages) print("--------------Result") diff --git a/postgres/postgres-init.sql b/postgres/postgres-init.sql index dc3ebbb..010e4c0 100644 --- a/postgres/postgres-init.sql +++ b/postgres/postgres-init.sql @@ -1,8 +1,62 @@ -CREATE USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD'; -ALTER USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD'; -GRANT ALL PRIVILEGES ON DATABASE :PG_DB TO :PG_USERNAME; -GRANT ALL PRIVILEGES ON SCHEMA public TO :PG_USERNAME; -GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO :PG_USERNAME; -GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO :PG_USERNAME; -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :PG_USERNAME; -ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :PG_USERNAME; +-- CREATE USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD'; +-- ALTER USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD'; +-- GRANT ALL PRIVILEGES ON DATABASE :PG_DB TO :PG_USERNAME; +-- GRANT ALL PRIVILEGES ON SCHEMA public TO :PG_USERNAME; +-- GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO :PG_USERNAME; +-- GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO :PG_USERNAME; +-- ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :PG_USERNAME; +-- ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :PG_USERNAME; + + +-- create databases +CREATE DATABASE :PG_DB_JOBS; +CREATE DATABASE :PG_DB_TIMESERIES; + +-- create roles +CREATE ROLE :JOBS_RW_USER_NAME LOGIN PASSWORD :'JOBS_RW_USER_PASSWORD'; +CREATE ROLE :TIMESERIES_RW_USER_NAME LOGIN PASSWORD :'TIMESERIES_RW_USER_PASSWORD'; +ALTER ROLE :TIMESERIES_RW_USER_NAME CREATEDB; +CREATE ROLE :TIMESERIES_RO_USER_NAME LOGIN PASSWORD :'TIMESERIES_RO_USER_PASSWORD'; + +-- DB1 +\connect :PG_DB_JOBS + +-- Make RW user owner +ALTER DATABASE :PG_DB_JOBS OWNER TO :JOBS_RW_USER_NAME; +ALTER SCHEMA public OWNER TO :JOBS_RW_USER_NAME; + +-- Ensure RW can create objects +GRANT ALL ON SCHEMA public TO :JOBS_RW_USER_NAME; + +-- Existing objects +GRANT ALL ON ALL TABLES IN SCHEMA public TO :JOBS_RW_USER_NAME; +GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO :JOBS_RW_USER_NAME; +GRANT ALL ON ALL FUNCTIONS IN SCHEMA public TO :JOBS_RW_USER_NAME; + +-- Default privileges +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :JOBS_RW_USER_NAME; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :JOBS_RW_USER_NAME; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON FUNCTIONS TO :JOBS_RW_USER_NAME; + +-- DB2 +\connect :PG_DB_TIMESERIES + +-- Make RW owner +ALTER DATABASE :PG_DB_TIMESERIES OWNER TO :TIMESERIES_RW_USER_NAME; +ALTER SCHEMA public OWNER TO :TIMESERIES_RW_USER_NAME; + +-- Schema privileges +GRANT ALL ON SCHEMA public TO :TIMESERIES_RW_USER_NAME; +GRANT USAGE ON SCHEMA public TO :TIMESERIES_RO_USER_NAME; + +-- Existing objects +GRANT ALL ON ALL TABLES IN SCHEMA public TO :TIMESERIES_RW_USER_NAME; +GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO :TIMESERIES_RW_USER_NAME; +GRANT ALL ON ALL FUNCTIONS IN SCHEMA public TO :TIMESERIES_RW_USER_NAME; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO :TIMESERIES_RO_USER_NAME; + +-- Default privileges +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :TIMESERIES_RW_USER_NAME; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :TIMESERIES_RW_USER_NAME; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON FUNCTIONS TO :TIMESERIES_RW_USER_NAME; +ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO :TIMESERIES_RO_USER_NAME; diff --git a/scripts/setup.ps1 b/scripts/setup.ps1 index 023f568..e2e4e62 100644 --- a/scripts/setup.ps1 +++ b/scripts/setup.ps1 @@ -13,21 +13,21 @@ get-content .env | foreach { # docker compose stop # Deploy postgres omotes schema -docker compose --profile=manual_dev down orchestrator_postgres_db_dev -docker compose up -d --wait orchestrator_postgres_db -docker compose exec orchestrator_postgres_db psql -d postgres -c 'CREATE DATABASE omotes_jobs;' -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "CREATE USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "ALTER USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON DATABASE omotes_jobs TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" -docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose down +docker compose up -d --wait postgres_db +docker compose exec omotes_postgres psql -d postgres -c 'CREATE DATABASE omotes_jobs;' +docker compose exec omotes_postgres psql -d omotes_jobs -c "CREATE USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" +docker compose exec omotes_postgres psql -d omotes_jobs -c "ALTER USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';" +docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON DATABASE omotes_jobs TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" +docker compose exec omotes_postgres psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;" # Upgrade omotes tables -docker compose build orchestrator_postgres_db_upgrade -docker compose run --rm orchestrator_postgres_db_upgrade +docker compose build omotes_postgres_upgrade +docker compose run --rm omotes_postgres_upgrade # Setup rabbitmq docker compose up -d --wait rabbitmq diff --git a/scripts/setup.sh b/scripts/setup.sh index 57ca0cf..a6c90f0 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -10,15 +10,22 @@ DOCKER_COMPOSE_FILE=${2:-"./docker-compose.yml -f ./docker-compose.override.setu echo "Using docker compose file at: $DOCKER_COMPOSE_FILE" # stop system before setup -$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE --profile=manual_dev down +$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE down # add postgres user with privileges -$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE up --wait orchestrator_postgres_db -$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE exec orchestrator_postgres_db psql \ - -d omotes_jobs \ - -v PG_USERNAME="$POSTGRES_ORCHESTRATOR_USER_NAME" \ - -v PG_PASSWORD="$POSTGRES_ORCHESTRATOR_USER_PASSWORD" \ - -v PG_DB=omotes_jobs \ +$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE up --wait omotes_postgres +$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE exec omotes_postgres psql \ + -U "$POSTGRES_ROOT_USER" \ + -p "$POSTGRES_PORT" \ + -d postgres \ + -v PG_DB_JOBS=omotes_jobs \ + -v JOBS_RW_USER_NAME="$POSTGRES_ORCHESTRATOR_USER_NAME" \ + -v JOBS_RW_USER_PASSWORD="$POSTGRES_ORCHESTRATOR_USER_PASSWORD" \ + -v PG_DB_TIMESERIES=omotes_timeseries \ + -v TIMESERIES_RW_USER_NAME="$POSTGRES_TIMESERIES_RW_USER_NAME" \ + -v TIMESERIES_RW_USER_PASSWORD="$POSTGRES_TIMESERIES_RW_USER_PASSWORD" \ + -v TIMESERIES_RO_USER_NAME="$POSTGRES_TIMESERIES_RO_USER_NAME" \ + -v TIMESERIES_RO_USER_PASSWORD="$POSTGRES_TIMESERIES_RO_USER_PASSWORD" \ -f /setup/init.sql # add rabbitmq 'omotes' and 'celery' vhosts and users diff --git a/scripts/start-dev.sh b/scripts/start-dev.sh index 8966e7c..9f7c31d 100755 --- a/scripts/start-dev.sh +++ b/scripts/start-dev.sh @@ -2,6 +2,6 @@ . scripts/_select_docker_compose.sh -$DOCKER_COMPOSE --profile=manual_dev down orchestrator_postgres_db_dev +$DOCKER_COMPOSE down $DOCKER_COMPOSE -f docker-compose.yml -f docker-compose.override.dev.yml build $DOCKER_COMPOSE -f docker-compose.yml -f docker-compose.override.dev.yml up --wait diff --git a/scripts/start.sh b/scripts/start.sh index 8592058..d963b44 100755 --- a/scripts/start.sh +++ b/scripts/start.sh @@ -2,5 +2,5 @@ . scripts/_select_docker_compose.sh -$DOCKER_COMPOSE --profile=manual_dev down orchestrator_postgres_db_dev +$DOCKER_COMPOSE down $DOCKER_COMPOSE up --wait diff --git a/scripts/start_postgres_in_dev_mode.sh b/scripts/start_postgres_in_dev_mode.sh deleted file mode 100755 index d899b06..0000000 --- a/scripts/start_postgres_in_dev_mode.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -docker compose -f docker-compose.yml down orchestrator_postgres_db -docker compose -f docker-compose.yml up --wait orchestrator_postgres_db_dev diff --git a/scripts/stop.sh b/scripts/stop.sh index b23ebf4..cfa898d 100755 --- a/scripts/stop.sh +++ b/scripts/stop.sh @@ -1,4 +1,4 @@ #!/bin/bash . scripts/_select_docker_compose.sh -$DOCKER_COMPOSE --profile=manual_dev down +$DOCKER_COMPOSE down diff --git a/system_tests/docker-compose.override.yml b/system_tests/docker-compose.override.yml index ba603c6..0cd1a94 100644 --- a/system_tests/docker-compose.override.yml +++ b/system_tests/docker-compose.override.yml @@ -17,7 +17,7 @@ services: RABBITMQ_HOST: rabbitmq RABBITMQ_PORT: 5672 - POSTGRES_HOST: orchestrator_postgres_db + POSTGRES_HOST: omotes_postgres POSTGRES_PORT: 5432 POSTGRES_ORCHESTRATOR_USER_NAME: omotes_orchestrator POSTGRES_ORCHESTRATOR_USER_PASSWORD: somepass3 @@ -32,6 +32,6 @@ services: TIMEOUT_JOB_HANDLER_RERUN_SEC: 5 JOB_RETENTION_SEC: 20 - orchestrator_postgres_db: + omotes_postgres: ports: - "${POSTGRES_DEV_PORT}:5432" diff --git a/track_memory.sh b/track_memory.sh new file mode 100755 index 0000000..46fdc1d --- /dev/null +++ b/track_memory.sh @@ -0,0 +1,4 @@ +while true; do + date +"%Y-%m-%d %H:%M:%S" + docker stats 79e4e234711e --no-stream --format "{{.Name}} {{.MemUsage}}" +done >> container_mem.log \ No newline at end of file