174 enable postgres timeseries storage

.env.template

@@ -1,8 +1,12 @@
 POSTGRES_ROOT_USER=root
 POSTGRES_ROOT_PASSWORD=1234
-POSTGRES_DEV_PORT=6432
+POSTGRES_PORT=6432
 POSTGRES_ORCHESTRATOR_USER_NAME=omotes_orchestrator
 POSTGRES_ORCHESTRATOR_USER_PASSWORD=somepass3
+POSTGRES_TIMESERIES_RW_USER_NAME=omotes_timeseries_rw
+POSTGRES_TIMESERIES_RW_USER_PASSWORD=somepass4
+POSTGRES_TIMESERIES_RO_USER_NAME=omotes_timeseries_ro
+POSTGRES_TIMESERIES_RO_USER_PASSWORD=somepass5
 
 RABBITMQ_PORT=5672
 RABBITMQ_MANAGEMENT_PORT=15672

.vscode/launch.json

@@ -0,0 +1,35 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Run optimizer.py",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${workspaceFolder}/example_sdk_client/optimizer.py",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}/example_sdk_client",
+            "justMyCode": true,
+            "env": {
+                "PYTHONPATH": "${workspaceFolder}/example_sdk_client"
+            }
+        },
+        {
+            "name": "test__simulator__multiple_ates_run",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "pytest",
+            "args": [
+                "test_workflows_steps.py::TestWorkflows::test__simulator__multiple_ates_run",
+                "-v",
+                "-s"
+            ],
+            "jinja": true,
+            "justMyCode": true,
+            "cwd": "${workspaceFolder}/system_tests/src",
+            "envFile": "${workspaceFolder}/.env.test",
+            "env": {
+                "PYTHONPATH": "${workspaceFolder}/system_tests"
+            }
+        }
+    ]
+}

docker-compose.override.dev.yml

@@ -1,10 +1,10 @@
 services:
 
-  orchestrator:
-    image: omotes-orchestrator-dev
-    build:
-      context: ..
-      dockerfile: orchestrator/dev.Dockerfile
+#  orchestrator:
+#    image: omotes-orchestrator-dev
+#    build:
+#      context: ..
+#      dockerfile: orchestrator/dev.Dockerfile
 
   grow_worker_optimizer: &grow_worker
     image: omotes-worker-mesido-dev
@@ -27,12 +27,8 @@ services:
   grow_worker_gurobi:
     <<: *grow_worker
 
-  omotes_simulator_worker:
-    image: omotes-worker-simulator-dev
-    build:
-      context: ..
-      dockerfile: simulator-worker/dev.Dockerfile
-
-  orchestrator_postgres_db:
-    ports:
-      - "${POSTGRES_DEV_PORT}:5432"
+#  omotes_simulator_worker:
+#    image: omotes-worker-simulator-dev
+#    build:
+#      context: ..
+#      dockerfile: simulator-worker/dev.Dockerfile

docker-compose.override.setup.yml

@@ -18,6 +18,6 @@ services:
       INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD: ${INFLUXDB_FRONTEND_ADMIN_USER_PASSWORD}
       INFLUXDB_PORT: ${INFLUXDB_PORT}
 
-  orchestrator_postgres_db:
+  omotes_postgres:
     volumes:
       - "${POSTGRES_INIT_FILE:-./postgres/postgres-init.sql}:/setup/init.sql:ro"

docker-compose.yml

@@ -60,16 +60,20 @@ services:
       - ./influxdb/influxdb.conf:/etc/influxdb/influxdb.conf:ro
       - influxdb_storage:/var/lib/influxdb
 
-  orchestrator_postgres_db: &orchestrator_postgres_db
+  omotes_postgres: &omotes_postgres
     image: postgres:15.4
     restart: unless-stopped
+    command: ["postgres", "-p", "${POSTGRES_PORT}"]
     volumes:
       - "db-data:/var/lib/postgresql/data/"
+    expose:
+      - "${POSTGRES_PORT}"
+    ports:
+      - "${POSTGRES_PORT}:${POSTGRES_PORT}"
     networks:
       - omotes
     environment:
       PGDATA: /var/lib/postgresql/data
-      POSTGRES_DB: omotes_jobs
       POSTGRES_USER: ${POSTGRES_ROOT_USER}
       POSTGRES_PASSWORD: ${POSTGRES_ROOT_PASSWORD}
 
@@ -81,22 +85,13 @@ services:
           "pg_isready",
           "-U",
           "${POSTGRES_ROOT_USER}",
-          "-d",
-          "omotes_jobs",
+          "-p",
+          "${POSTGRES_PORT}"
         ]
       interval: 5s
       timeout: 5s
       retries: 10
 
-  # Skipped in 'docker-compose up', only used to develop database revisions.
-  orchestrator_postgres_db_dev:
-    <<: *orchestrator_postgres_db
-    hostname: orchestrator_postgres_db
-    ports:
-      - "${POSTGRES_DEV_PORT}:5432"
-    profiles:
-      - manual_dev
-
   orchestrator:
     image: ghcr.io/project-omotes/omotes_orchestrator:1.2.6
     restart: unless-stopped
@@ -123,8 +118,8 @@ services:
       TASK_RABBITMQ_PORT: 5672
       TASK_RABBITMQ_VIRTUALHOST: omotes_celery
 
-      POSTGRESQL_HOST: orchestrator_postgres_db
-      POSTGRESQL_PORT: 5432
+      POSTGRESQL_HOST: omotes_postgres
+      POSTGRESQL_PORT: ${POSTGRES_PORT}
       POSTGRESQL_DATABASE: omotes_jobs
       POSTGRESQL_USERNAME: ${POSTGRES_ORCHESTRATOR_USER_NAME}
       POSTGRESQL_PASSWORD: ${POSTGRES_ORCHESTRATOR_USER_PASSWORD}
@@ -143,13 +138,13 @@ services:
     depends_on:
       rabbitmq:
         condition: service_healthy
-      orchestrator_postgres_db:
+      omotes_postgres:
         condition: service_healthy
       omotes_influxdb:
         condition: service_healthy
 
   grow_worker_optimizer: &grow_worker
-    image: ghcr.io/project-omotes/omotes-grow-optimizer-worker:2.1.3
+    image: ghcr.io/project-omotes/omotes-grow-optimizer-worker:2.1.7
     restart: unless-stopped
     deploy: &grow_worker_deploy
       replicas: 2

example_sdk_client/optimizer.py

@@ -11,17 +11,17 @@
     JobStatusUpdate,
 )
 
-rabbitmq_config = RabbitMQConfig(username="omotes", password="somepass1", virtual_host="omotes")
+rabbitmq_config = RabbitMQConfig(port=5673,username="omotes", password="somepass1", virtual_host="omotes")
 STOP_EVENT = threading.Event()
 
 
 def handle_on_finished(job: Job, result: JobResult):
-    print("--------------Logs:")
-    print(result.logs)
-    print()
-    print("--------------ESDL:")
-    print(result.output_esdl)
-    print()
+    # print("--------------Logs:")
+    # print(result.logs)
+    # print()
+    # print("--------------ESDL:")
+    # print(result.output_esdl)
+    # print()
     print("--------------ESDL messages:")
     print(result.esdl_messages)
     print("--------------Result")

postgres/postgres-init.sql

@@ -1,8 +1,62 @@
-CREATE USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD';
-ALTER USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD';
-GRANT ALL PRIVILEGES ON DATABASE :PG_DB TO :PG_USERNAME;
-GRANT ALL PRIVILEGES ON SCHEMA public TO :PG_USERNAME;
-GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO :PG_USERNAME;
-GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO :PG_USERNAME;
-ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :PG_USERNAME;
-ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :PG_USERNAME;
+-- CREATE USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD';
+-- ALTER USER :PG_USERNAME WITH PASSWORD :'PG_PASSWORD';
+-- GRANT ALL PRIVILEGES ON DATABASE :PG_DB TO :PG_USERNAME;
+-- GRANT ALL PRIVILEGES ON SCHEMA public TO :PG_USERNAME;
+-- GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO :PG_USERNAME;
+-- GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO :PG_USERNAME;
+-- ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :PG_USERNAME;
+-- ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :PG_USERNAME;
+
+
+-- create databases
+CREATE DATABASE :PG_DB_JOBS;
+CREATE DATABASE :PG_DB_TIMESERIES;
+
+-- create roles
+CREATE ROLE :JOBS_RW_USER_NAME LOGIN PASSWORD :'JOBS_RW_USER_PASSWORD';
+CREATE ROLE :TIMESERIES_RW_USER_NAME LOGIN PASSWORD :'TIMESERIES_RW_USER_PASSWORD';
+ALTER ROLE :TIMESERIES_RW_USER_NAME CREATEDB;
+CREATE ROLE :TIMESERIES_RO_USER_NAME LOGIN PASSWORD :'TIMESERIES_RO_USER_PASSWORD';
+
+-- DB1
+\connect :PG_DB_JOBS
+
+-- Make RW user owner
+ALTER DATABASE :PG_DB_JOBS OWNER TO :JOBS_RW_USER_NAME;
+ALTER SCHEMA public OWNER TO :JOBS_RW_USER_NAME;
+
+-- Ensure RW can create objects
+GRANT ALL ON SCHEMA public TO :JOBS_RW_USER_NAME;
+
+-- Existing objects
+GRANT ALL ON ALL TABLES IN SCHEMA public TO :JOBS_RW_USER_NAME;
+GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO :JOBS_RW_USER_NAME;
+GRANT ALL ON ALL FUNCTIONS IN SCHEMA public TO :JOBS_RW_USER_NAME;
+
+-- Default privileges
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :JOBS_RW_USER_NAME;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :JOBS_RW_USER_NAME;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON FUNCTIONS TO :JOBS_RW_USER_NAME;
+
+-- DB2
+\connect :PG_DB_TIMESERIES
+
+-- Make RW owner
+ALTER DATABASE :PG_DB_TIMESERIES OWNER TO :TIMESERIES_RW_USER_NAME;
+ALTER SCHEMA public OWNER TO :TIMESERIES_RW_USER_NAME;
+
+-- Schema privileges
+GRANT ALL ON SCHEMA public TO :TIMESERIES_RW_USER_NAME;
+GRANT USAGE ON SCHEMA public TO :TIMESERIES_RO_USER_NAME;
+
+-- Existing objects
+GRANT ALL ON ALL TABLES IN SCHEMA public TO :TIMESERIES_RW_USER_NAME;
+GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO :TIMESERIES_RW_USER_NAME;
+GRANT ALL ON ALL FUNCTIONS IN SCHEMA public TO :TIMESERIES_RW_USER_NAME;
+GRANT SELECT ON ALL TABLES IN SCHEMA public TO :TIMESERIES_RO_USER_NAME;
+
+-- Default privileges
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO :TIMESERIES_RW_USER_NAME;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO :TIMESERIES_RW_USER_NAME;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON FUNCTIONS TO :TIMESERIES_RW_USER_NAME;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO :TIMESERIES_RO_USER_NAME;

scripts/setup.ps1

@@ -13,21 +13,21 @@ get-content .env | foreach {
 # docker compose stop
 
 # Deploy postgres omotes schema
-docker compose --profile=manual_dev down orchestrator_postgres_db_dev
-docker compose up -d --wait orchestrator_postgres_db
-docker compose exec orchestrator_postgres_db psql -d postgres -c 'CREATE DATABASE omotes_jobs;'
-docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "CREATE USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';"
-docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "ALTER USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';"
-docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON DATABASE omotes_jobs TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
-docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
-docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
-docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
-docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
-docker compose exec orchestrator_postgres_db psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
+docker compose down
+docker compose up -d --wait postgres_db
+docker compose exec omotes_postgres psql -d postgres -c 'CREATE DATABASE omotes_jobs;'
+docker compose exec omotes_postgres psql -d omotes_jobs -c "CREATE USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';"
+docker compose exec omotes_postgres psql -d omotes_jobs -c "ALTER USER $env:POSTGRES_ORCHESTRATOR_USER_NAME WITH PASSWORD '$env:POSTGRES_ORCHESTRATOR_USER_PASSWORD';"
+docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON DATABASE omotes_jobs TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
+docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
+docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
+docker compose exec omotes_postgres psql -d omotes_jobs -c "GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
+docker compose exec omotes_postgres psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
+docker compose exec omotes_postgres psql -d omotes_jobs -c "ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO $env:POSTGRES_ORCHESTRATOR_USER_NAME;"
 
 # Upgrade omotes tables
-docker compose build orchestrator_postgres_db_upgrade
-docker compose run --rm orchestrator_postgres_db_upgrade
+docker compose build omotes_postgres_upgrade
+docker compose run --rm omotes_postgres_upgrade
 
 # Setup rabbitmq
 docker compose up -d --wait rabbitmq

scripts/setup.sh

@@ -10,15 +10,22 @@ DOCKER_COMPOSE_FILE=${2:-"./docker-compose.yml -f ./docker-compose.override.setu
 echo "Using docker compose file at: $DOCKER_COMPOSE_FILE"
 
 # stop system before setup
-$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE --profile=manual_dev down
+$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE down
 
 # add postgres user with privileges
-$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE up --wait orchestrator_postgres_db
-$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE exec orchestrator_postgres_db psql \
-  -d omotes_jobs \
-  -v PG_USERNAME="$POSTGRES_ORCHESTRATOR_USER_NAME" \
-  -v PG_PASSWORD="$POSTGRES_ORCHESTRATOR_USER_PASSWORD" \
-  -v PG_DB=omotes_jobs \
+$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE --env-file $ENV_FILE up --wait omotes_postgres
+$DOCKER_COMPOSE -f $DOCKER_COMPOSE_FILE exec omotes_postgres psql \
+  -U "$POSTGRES_ROOT_USER" \
+  -p "$POSTGRES_PORT" \
+  -d postgres \
+  -v PG_DB_JOBS=omotes_jobs \
+  -v JOBS_RW_USER_NAME="$POSTGRES_ORCHESTRATOR_USER_NAME" \
+  -v JOBS_RW_USER_PASSWORD="$POSTGRES_ORCHESTRATOR_USER_PASSWORD" \
+  -v PG_DB_TIMESERIES=omotes_timeseries \
+  -v TIMESERIES_RW_USER_NAME="$POSTGRES_TIMESERIES_RW_USER_NAME" \
+  -v TIMESERIES_RW_USER_PASSWORD="$POSTGRES_TIMESERIES_RW_USER_PASSWORD" \
+  -v TIMESERIES_RO_USER_NAME="$POSTGRES_TIMESERIES_RO_USER_NAME" \
+  -v TIMESERIES_RO_USER_PASSWORD="$POSTGRES_TIMESERIES_RO_USER_PASSWORD" \
   -f /setup/init.sql
 
 # add rabbitmq 'omotes' and 'celery' vhosts and users

scripts/start-dev.sh

@@ -2,6 +2,6 @@
 
 . scripts/_select_docker_compose.sh
 
-$DOCKER_COMPOSE --profile=manual_dev down orchestrator_postgres_db_dev
+$DOCKER_COMPOSE down
 $DOCKER_COMPOSE -f docker-compose.yml -f docker-compose.override.dev.yml build
 $DOCKER_COMPOSE -f docker-compose.yml -f docker-compose.override.dev.yml up --wait

scripts/start.sh

@@ -2,5 +2,5 @@
 
 . scripts/_select_docker_compose.sh
 
-$DOCKER_COMPOSE --profile=manual_dev down orchestrator_postgres_db_dev
+$DOCKER_COMPOSE down
 $DOCKER_COMPOSE up --wait

scripts/stop.sh

@@ -1,4 +1,4 @@
 #!/bin/bash
 . scripts/_select_docker_compose.sh
 
-$DOCKER_COMPOSE --profile=manual_dev down
+$DOCKER_COMPOSE down

system_tests/docker-compose.override.yml

@@ -17,7 +17,7 @@ services:
       RABBITMQ_HOST: rabbitmq
       RABBITMQ_PORT: 5672
 
-      POSTGRES_HOST: orchestrator_postgres_db
+      POSTGRES_HOST: omotes_postgres
       POSTGRES_PORT: 5432
       POSTGRES_ORCHESTRATOR_USER_NAME: omotes_orchestrator
       POSTGRES_ORCHESTRATOR_USER_PASSWORD: somepass3
@@ -32,6 +32,6 @@ services:
       TIMEOUT_JOB_HANDLER_RERUN_SEC: 5
       JOB_RETENTION_SEC: 20
 
-  orchestrator_postgres_db:
+  omotes_postgres:
     ports:
       - "${POSTGRES_DEV_PORT}:5432"

track_memory.sh

@@ -0,0 +1,4 @@
+while true; do
+    date +"%Y-%m-%d %H:%M:%S"
+    docker stats 79e4e234711e --no-stream --format "{{.Name}} {{.MemUsage}}"
+done >> container_mem.log