chore: make toy tutorials robust against pre-existing IDs

.github/workflows/docker-build.yml

@@ -47,27 +47,27 @@ jobs:
           db upgrade
       - name: Add toy user
         run: docker exec --env-file .env fm-container flexmeasures
-          add toy-account
+          add toy-account --kind battery --shell-vars | grep '^FM_TOY_' >> $GITHUB_ENV
       - name: Generate prices dummy data
         run: ci/generate-dummy-price.sh
       - name: Copy prices dummy data
         run: docker cp prices-tomorrow.csv fm-container:/app/prices-tomorrow.csv
       - name: Add beliefs
         run: |
           docker exec --env-file .env fm-container flexmeasures \
-            add beliefs --sensor 1 --source toy-user prices-tomorrow.csv --timezone Europe/Amsterdam
+            add beliefs --sensor ${FM_TOY_PRICE_SENSOR_ID} --source toy-user prices-tomorrow.csv --timezone Europe/Amsterdam
       - name: Export TOMORROW
         run: echo "TOMORROW=$(date --date="next day" '+%Y-%m-%d')"
           >> $GITHUB_ENV
       - name: Add schedule
         run: |
           docker exec --env-file .env fm-container flexmeasures \
-            add schedule --sensor 2 --start ${TOMORROW}T07:00+01:00 \
+            add schedule --sensor ${FM_TOY_BATTERY_SENSOR_ID} --start ${TOMORROW}T07:00+01:00 \
             --duration PT12H --soc-at-start 50% --flex-model '{"roundtrip-efficiency": "90%"}'
       - name: Add Toy Account for process
-        run: docker exec --env-file .env fm-container flexmeasures add toy-account --kind process
+        run: docker exec --env-file .env fm-container flexmeasures add toy-account --kind process --shell-vars | grep '^FM_TOY_' >> $GITHUB_ENV
       - name: Add Process schedule
         run: |
-          docker exec --env-file .env fm-container flexmeasures add schedule --sensor 5 --scheduler ProcessScheduler \
-            --start ${TOMORROW}T00:00:00+02:00 --duration PT24H --flex-context '{"consumption-price": {"sensor": 1}}' \
+          docker exec --env-file .env fm-container flexmeasures add schedule --sensor ${FM_TOY_PROCESS_BREAKABLE_SENSOR_ID} --scheduler ProcessScheduler \
+            --start ${TOMORROW}T00:00:00+02:00 --duration PT24H --flex-context '{"consumption-price": {"sensor": '"${FM_TOY_PRICE_SENSOR_ID}"'}}' \
             --flex-model "{\"duration\": \"PT4H\", \"process-type\": \"BREAKABLE\", \"power\": 0.2, \"time-restrictions\": [{\"start\": \"${TOMORROW}T15:00:00+02:00\", \"duration\": \"PT1H\"}]}"

documentation/changelog.rst

@@ -29,6 +29,7 @@ Infrastructure / Support
 * Document the ``TRUSTED_HOSTS`` setting to safeguard against host poisoning from clients [see `PR #2246 <https://www.github.com/FlexMeasures/flexmeasures/pull/2246>`_]
 * Document the various ways to inspect a (scheduling) job [see `PR #2247 <https://www.github.com/FlexMeasures/flexmeasures/pull/2247>`_]
 * Automate Docker Hub image publishing and a PyPI install smoke test on release, add a manually-triggered QA workflow that runs the toy tutorials and HEMS walkthrough against a local Docker Compose stack, and add a helper script to list merged PRs since the last tag [see `PR #2260 <https://www.github.com/FlexMeasures/flexmeasures/pull/2260>`_]
+* Make toy tutorials robust against pre-existing IDs [see `PR #2269 <https://www.github.com/FlexMeasures/flexmeasures/pull/2269>`_]
 
 Bugfixes
 -----------

documentation/dev/docker-compose.rst

@@ -99,7 +99,7 @@ Next, we put a scheduling job in the worker's queue. This only works because we
 
 .. code-block:: bash
 
-    $ flexmeasures add schedule --sensor 2 \
+    $ flexmeasures add schedule --sensor ${FM_TOY_BATTERY_SENSOR_ID} \
         --start ${TOMORROW}T07:00+01:00 --duration PT12H --soc-at-start 50% \
         --flex-model '{"soc-min": "50 kWh"}' --as-job
 
@@ -116,7 +116,7 @@ We'll not go into the server container this time, but simply send a command:
 .. code-block:: bash
 
     $ TOMORROW=$(date --date="next day" '+%Y-%m-%d')
-    $ docker exec -it flexmeasures-server-1 bash -c "flexmeasures show beliefs --sensor 2 --start ${TOMORROW}T07:00:00+01:00 --duration PT12H"
+    $ docker exec -it flexmeasures-server-1 bash -c "flexmeasures show beliefs --sensor ${FM_TOY_BATTERY_SENSOR_ID} --start ${TOMORROW}T07:00:00+01:00 --duration PT12H"
 
 The charging/discharging schedule should be there:
 

documentation/index.rst

@@ -53,13 +53,13 @@ The main purpose of FlexMeasures is to create optimized schedules. Let's have a
             $ docker pull postgres; docker run --name pg-docker -e POSTGRES_PASSWORD=docker -e POSTGRES_DB=flexmeasures-db -d -p 5433:5432 postgres:latest 
             $ export SQLALCHEMY_DATABASE_URI="postgresql://postgres:docker@127.0.0.1:5433/flexmeasures-db" && export SECRET_KEY=notsecret && export SECURITY_TOTP_SECRETS={"1":"something-secret"}
             $ flexmeasures db upgrade  # create tables
-            $ flexmeasures add toy-account --kind battery  # setup account incl. a user, battery (ID 2) and market (ID 1)
-            $ flexmeasures add beliefs --sensor 2 --source toy-user prices-tomorrow.csv --timezone utc  # load prices, also possible per API
-            $ flexmeasures add schedule --sensor 2 \
+            $ eval "$(flexmeasures add toy-account --kind battery --shell-vars | grep '^FM_TOY_')"  # setup account incl. a user, battery, solar and market
+            $ flexmeasures add beliefs --sensor ${FM_TOY_PRICE_SENSOR_ID} --source toy-user prices-tomorrow.csv --timezone utc  # load prices, also possible per API
+            $ flexmeasures add schedule --sensor ${FM_TOY_BATTERY_SENSOR_ID} \
                 --start ${TOMORROW}T07:00+01:00 --duration PT12H \
-                --soc-at-start 50% --flex-context '{"consumption-price": {"sensor": 1}}' \
+                --soc-at-start 50% --flex-context '{"consumption-price": {"sensor": '"${FM_TOY_PRICE_SENSOR_ID}"'}}' \
                 --flex-model '{"roundtrip-efficiency": "90%"}' # this is also possible per API
-            $ flexmeasures show beliefs --sensor 2 --start ${TOMORROW}T07:00:00+01:00 --duration PT12H  # also visible per UI, of course
+            $ flexmeasures show beliefs --sensor ${FM_TOY_BATTERY_SENSOR_ID} --start ${TOMORROW}T07:00:00+01:00 --duration PT12H  # also visible per UI, of course
 
 
 A short explanation of the optimization shown above: This battery is optimized to buy power cheaply and sell it at expensive times - the red-dotted line is what FlexMeasures computed to be the best schedule, given all knowledge (in this case, the prices shown in blue). However, in the example in the middle tab, the battery has to store local solar power as well (orange line), which constrains how much it can do with its capacity (that's why the schedule is limited in capacity and thus cycling less energy overall than on the left).

documentation/tut/building_uis.rst

@@ -284,7 +284,7 @@ Now let's call this function when the HTML page is opened, to embed our chart:
     }
 
 The parameters we pass in describe what we want to see: all data for sensor 3 since 2022.
-If you followed our :ref:`toy tutorial<tut_toy_schedule>` on a fresh FlexMeasures installation, sensor 1 contains market prices (authenticate with the toy-user to gain access).
+If you followed our :ref:`toy tutorial<tut_toy_schedule>` on a fresh FlexMeasures installation, ``FM_TOY_PRICE_SENSOR_ID`` contains the market-price sensor ID (authenticate with the toy-user to gain access).
 
 
 The result looks like this in your browser:

documentation/tut/scripts/run-tutorial-in-docker.sh

@@ -6,6 +6,8 @@ CONTAINER_NAME="${1:-$(basename $(pwd))-server-1}"
 echo "[TUTORIAL-RUNNER] RUNNING TUTORIAL 1 (SIMPLE BATTERY SCHEDULE)..."
 echo "-----------------------------------------------------------------"
 
+eval "$(docker exec -i $CONTAINER_NAME flexmeasures add toy-account --kind battery --shell-vars | grep '^FM_TOY_')"
+
 echo "[TUTORIAL-RUNNER] loading prices..."
 TOMORROW=$(date --date="next day" '+%Y-%m-%d')
 echo "Hour,Price
@@ -37,14 +39,14 @@ ${TOMORROW}T23:00:00,7" > prices-tomorrow.csv
 docker cp prices-tomorrow.csv $CONTAINER_NAME:/app
 
 docker exec -it $CONTAINER_NAME flexmeasures add beliefs \
-  --sensor 1 --source toy-user /app/prices-tomorrow.csv --timezone Europe/Amsterdam
+  --sensor ${FM_TOY_PRICE_SENSOR_ID} --source toy-user /app/prices-tomorrow.csv --timezone Europe/Amsterdam
 
 echo "[TUTORIAL-RUNNER] creating schedule ..."
 docker exec -it $CONTAINER_NAME flexmeasures add schedule \
-  --sensor 2 \
+  --sensor ${FM_TOY_BATTERY_SENSOR_ID} \
   --start ${TOMORROW}T07:00+01:00 --duration PT12H --soc-at-start 50% \
   --flex-model '{"soc-min": "50 kWh"}'
 
 echo "[TUTORIAL-RUNNER] displaying schedule..."
 docker exec -it $CONTAINER_NAME flexmeasures show beliefs \
-  --sensor 2 --start ${TOMORROW}T07:00:00+01:00 --duration PT12H
+  --sensor ${FM_TOY_BATTERY_SENSOR_ID} --start ${TOMORROW}T07:00:00+01:00 --duration PT12H

documentation/tut/scripts/run-tutorial2-in-docker.sh

@@ -6,6 +6,8 @@ CONTAINER_NAME="${1:-$(basename $(pwd))-server-1}"
 echo "[TUTORIAL-RUNNER] RUNNING TUTORIAL 2 (ADDING SOLAR FORECAST) ..."
 echo "------------------------------------------------------------"
 
+eval "$(docker exec -i $CONTAINER_NAME flexmeasures add toy-account --kind battery --shell-vars | grep '^FM_TOY_')"
+
 echo "[TUTORIAL-RUNNER] loading solar production data..."
 
 TOMORROW=$(date --date="next day" '+%Y-%m-%d')
@@ -42,18 +44,18 @@ echo "[TUTORIAL-RUNNER] adding source ..."
 docker exec -it $CONTAINER_NAME flexmeasures add source --name "toy-forecaster" --type forecaster
 
 echo "[TUTORIAL-RUNNER] adding beliefs ..."
-docker exec -it $CONTAINER_NAME flexmeasures add beliefs --sensor 3 --source 4 /app/solar-tomorrow.csv --timezone Europe/Amsterdam
+docker exec -it $CONTAINER_NAME flexmeasures add beliefs --sensor ${FM_TOY_SOLAR_SENSOR_ID} --source 4 /app/solar-tomorrow.csv --timezone Europe/Amsterdam
 echo "[TUTORIAL-RUNNER] showing beliefs ..."
-docker exec -it $CONTAINER_NAME flexmeasures show beliefs --sensor 3 --start ${TOMORROW}T07:00:00+01:00 --duration PT12H
+docker exec -it $CONTAINER_NAME flexmeasures show beliefs --sensor ${FM_TOY_SOLAR_SENSOR_ID} --start ${TOMORROW}T07:00:00+01:00 --duration PT12H
 
 echo "[TUTORIAL-RUNNER] update schedule taking solar into account ..."
-docker exec -it $CONTAINER_NAME flexmeasures add schedule --sensor 2 \
+docker exec -it $CONTAINER_NAME flexmeasures add schedule --sensor ${FM_TOY_BATTERY_SENSOR_ID} \
   --start ${TOMORROW}T07:00+01:00 --duration PT12H --soc-at-start 50% \
-  --flex-context '{"inflexible-device-sensors": [3]}' \
+  --flex-context '{"inflexible-device-sensors": ['"${FM_TOY_SOLAR_SENSOR_ID}"']}' \
   --flex-model '{"soc-min": "50 kWh"}'
 
 echo "[TUTORIAL-RUNNER] showing schedule ..."
-docker exec -it $CONTAINER_NAME flexmeasures show beliefs --sensor 2 --start ${TOMORROW}T07:00:00+01:00 --duration PT12H
+docker exec -it $CONTAINER_NAME flexmeasures show beliefs --sensor ${FM_TOY_BATTERY_SENSOR_ID} --start ${TOMORROW}T07:00:00+01:00 --duration PT12H
 
 #echo ""
 #echo "[TUTORIAL-RUNNER] DEMONSTRATING CUSTOM SCHEDULING RESOLUTION ..."
@@ -63,12 +65,12 @@ docker exec -it $CONTAINER_NAME flexmeasures show beliefs --sensor 2 --start ${T
 #echo ""
 #
 #echo "[TUTORIAL-RUNNER] creating hourly-resolution schedule ..."
-#docker exec -it $CONTAINER_NAME flexmeasures add schedule --sensor 2 \
+#docker exec -it $CONTAINER_NAME flexmeasures add schedule --sensor ${FM_TOY_BATTERY_SENSOR_ID} \
 #  --start ${TOMORROW}T07:00+01:00 --duration PT12H --soc-at-start 50% \
 #  --resolution PT1H \
-#  --flex-context '{"inflexible-device-sensors": [3]}' \
+#  --flex-context '{"inflexible-device-sensors": ['"${FM_TOY_SOLAR_SENSOR_ID}"']}' \
 #  --flex-model '{"soc-min": "50 kWh"}'
 #
 #echo "[TUTORIAL-RUNNER] showing hourly-resolution schedule ..."
 #echo "[TUTORIAL-RUNNER] Notice the schedule still has 15-minute data points, but values only change each hour."
-#docker exec -it $CONTAINER_NAME flexmeasures show beliefs --sensor 2 --start ${TOMORROW}T07:00:00+01:00 --duration PT12H
+#docker exec -it $CONTAINER_NAME flexmeasures show beliefs --sensor ${FM_TOY_BATTERY_SENSOR_ID} --start ${TOMORROW}T07:00:00+01:00 --duration PT12H

documentation/tut/scripts/run-tutorial3-in-docker.sh

@@ -7,6 +7,7 @@ CONTAINER_NAME="${1:-$(basename $(pwd))-server-1}"
 echo "[TUTORIAL-RUNNER] RUNNING TUTORIAL 3 (PV CURTAILMENT / MORE THAN ONE FLEXIBLE ASSET) ..."
 echo "----------------------------------------------------------------------------------------"
 
+eval "$(docker exec -i $CONTAINER_NAME flexmeasures add toy-account --kind battery --shell-vars | grep '^FM_TOY_')"
 
 TOMORROW=$(date --date="next day" '+%Y-%m-%d')
 
@@ -25,34 +26,34 @@ echo '''{
 docker cp tutorial3-priceprofile-flex-context.json $CONTAINER_NAME:/app/ 
 
 # Running only the PV sensor
-docker exec -it $CONTAINER_NAME flexmeasures add schedule --sensor 3 \
+docker exec -it $CONTAINER_NAME flexmeasures add schedule --sensor ${FM_TOY_SOLAR_SENSOR_ID} \
   --start ${TOMORROW}T07:00+01:00 --duration PT12H \
-  --flex-model '{"consumption-capacity": "0 kW", "production-capacity": {"sensor": 3, "source": 4}}'\
+  --flex-model '{"consumption-capacity": "0 kW", "production-capacity": {"sensor": '"${FM_TOY_SOLAR_SENSOR_ID}"', "source": 4}}'\
   --flex-context tutorial3-priceprofile-flex-context.json 
 echo "[TUTORIAL-RUNNER] showing PV schedule ..."
-docker exec -it $CONTAINER_NAME flexmeasures show beliefs --sensor 3 --start ${TOMORROW}T07:00:00+01:00 --duration PT12H
+docker exec -it $CONTAINER_NAME flexmeasures show beliefs --sensor ${FM_TOY_SOLAR_SENSOR_ID} --start ${TOMORROW}T07:00:00+01:00 --duration PT12H
 
 echo "[TUTORIAL-RUNNER] Cleaning solar data for the next steps ..."
 # remove all previous beliefs on PV sensor so we don't have schedules mixed in the next run (issue 1807 can help with this, so selection by source works)
-docker exec -it $CONTAINER_NAME flexmeasures delete beliefs --sensor 3 --force
-docker exec -it $CONTAINER_NAME flexmeasures add beliefs --sensor 3 --source 4 /app/solar-tomorrow.csv --timezone Europe/Amsterdam
+docker exec -it $CONTAINER_NAME flexmeasures delete beliefs --sensor ${FM_TOY_SOLAR_SENSOR_ID} --force
+docker exec -it $CONTAINER_NAME flexmeasures add beliefs --sensor ${FM_TOY_SOLAR_SENSOR_ID} --source 4 /app/solar-tomorrow.csv --timezone Europe/Amsterdam
 
 echo "[TUTORIAL-RUNNER] Now running both battery and PV together, still using block price profiles ..."
-docker exec -it $CONTAINER_NAME flexmeasures add schedule --asset 2 \
+docker exec -it $CONTAINER_NAME flexmeasures add schedule --asset ${FM_TOY_BUILDING_ASSET_ID} \
   --start ${TOMORROW}T07:00+01:00 --duration PT12H \
-  --flex-model '[{"sensor": 3, "consumption-capacity": "0 kW", "production-capacity": {"sensor": 3, "source": 4}}, {"sensor": 2, "soc-at-start": "225 kWh", "soc-min": "50 kWh"}]'\
+  --flex-model '[{"sensor": '"${FM_TOY_SOLAR_SENSOR_ID}"', "consumption-capacity": "0 kW", "production-capacity": {"sensor": '"${FM_TOY_SOLAR_SENSOR_ID}"', "source": 4}}, {"sensor": '"${FM_TOY_BATTERY_SENSOR_ID}"', "soc-at-start": "225 kWh", "soc-min": "50 kWh"}]'\
   --flex-context tutorial3-priceprofile-flex-context.json 
 
 echo "[TUTORIAL-RUNNER] showing PV and battery schedule ..."
-docker exec -it $CONTAINER_NAME flexmeasures show beliefs --sensor 3 --sensor 2 --start ${TOMORROW}T07:00:00+01:00 --duration PT12H
+docker exec -it $CONTAINER_NAME flexmeasures show beliefs --sensor ${FM_TOY_SOLAR_SENSOR_ID} --sensor ${FM_TOY_BATTERY_SENSOR_ID} --start ${TOMORROW}T07:00:00+01:00 --duration PT12H
 
-docker exec -it $CONTAINER_NAME flexmeasures delete beliefs --sensor 3 --force
-docker exec -it $CONTAINER_NAME flexmeasures add beliefs --sensor 3 --source 4 /app/solar-tomorrow.csv --timezone Europe/Amsterdam
+docker exec -it $CONTAINER_NAME flexmeasures delete beliefs --sensor ${FM_TOY_SOLAR_SENSOR_ID} --force
+docker exec -it $CONTAINER_NAME flexmeasures add beliefs --sensor ${FM_TOY_SOLAR_SENSOR_ID} --source 4 /app/solar-tomorrow.csv --timezone Europe/Amsterdam
 
 echo "[TUTORIAL-RUNNER] Now running both battery and PV together, with realistic DA prices and larger battery ..."
-docker exec -it $CONTAINER_NAME flexmeasures add schedule --asset 2 \
+docker exec -it $CONTAINER_NAME flexmeasures add schedule --asset ${FM_TOY_BUILDING_ASSET_ID} \
   --start ${TOMORROW}T07:00+01:00 --duration PT12H \
-  --flex-model '[{"sensor": 3, "consumption-capacity": "0 kW", "production-capacity": {"sensor": 3, "source": 4}}, {"sensor": 2, "soc-at-start": "225 kWh", "soc-min": "50 kWh", "soc-max": "900kWh"}]'
+  --flex-model '[{"sensor": '"${FM_TOY_SOLAR_SENSOR_ID}"', "consumption-capacity": "0 kW", "production-capacity": {"sensor": '"${FM_TOY_SOLAR_SENSOR_ID}"', "source": 4}}, {"sensor": '"${FM_TOY_BATTERY_SENSOR_ID}"', "soc-at-start": "225 kWh", "soc-min": "50 kWh", "soc-max": "900kWh"}]'
 
 echo "[TUTORIAL-RUNNER] showing PV and battery schedule ..."
-docker exec -it $CONTAINER_NAME flexmeasures show beliefs --sensor 3 --sensor 2 --start ${TOMORROW}T07:00:00+01:00 --duration PT12H
+docker exec -it $CONTAINER_NAME flexmeasures show beliefs --sensor ${FM_TOY_SOLAR_SENSOR_ID} --sensor ${FM_TOY_BATTERY_SENSOR_ID} --start ${TOMORROW}T07:00:00+01:00 --duration PT12H

documentation/tut/scripts/run-tutorial4-in-docker.sh

@@ -9,22 +9,22 @@ echo "------------------------------------------------------------"
 TOMORROW=$(date --date="next day" '+%Y-%m-%d')
 
 echo "[TUTORIAL-RUNNER] Setting up toy account with reporters..."
-docker exec -it $CONTAINER_NAME flexmeasures add toy-account --kind process
+eval "$(docker exec -i $CONTAINER_NAME flexmeasures add toy-account --kind process --shell-vars | grep '^FM_TOY_')"
 
 echo "[TUTORIAL-RUNNER] Creating three process schedules ..."
-docker exec -it $CONTAINER_NAME flexmeasures add schedule --sensor 4 --scheduler ProcessScheduler \
+docker exec -it $CONTAINER_NAME flexmeasures add schedule --sensor ${FM_TOY_PROCESS_INFLEXIBLE_SENSOR_ID} --scheduler ProcessScheduler \
   --start ${TOMORROW}T00:00:00+02:00 --duration PT24H \
-  --flex-context '{"consumption-price": {"sensor": 1}}' \
+  --flex-context '{"consumption-price": {"sensor": '"${FM_TOY_PRICE_SENSOR_ID}"'}}' \
   --flex-model '{"duration": "PT4H", "process-type": "INFLEXIBLE", "power": 0.2, "time-restrictions": [{"start": "'"${TOMORROW}"'T15:00:00+02:00", "duration": "PT1H"}]}'
 
-docker exec -it $CONTAINER_NAME flexmeasures add schedule --sensor 5 --scheduler ProcessScheduler \
+docker exec -it $CONTAINER_NAME flexmeasures add schedule --sensor ${FM_TOY_PROCESS_BREAKABLE_SENSOR_ID} --scheduler ProcessScheduler \
   --start ${TOMORROW}T00:00:00+02:00 --duration PT24H \
-  --flex-context '{"consumption-price": {"sensor": 1}}' \
+  --flex-context '{"consumption-price": {"sensor": '"${FM_TOY_PRICE_SENSOR_ID}"'}}' \
   --flex-model '{"duration": "PT4H", "process-type": "BREAKABLE", "power": 0.2, "time-restrictions": [{"start": "'"${TOMORROW}"'T15:00:00+02:00", "duration": "PT1H"}]}'
 
-docker exec -it $CONTAINER_NAME flexmeasures add schedule --sensor 6 --scheduler ProcessScheduler \
+docker exec -it $CONTAINER_NAME flexmeasures add schedule --sensor ${FM_TOY_PROCESS_SHIFTABLE_SENSOR_ID} --scheduler ProcessScheduler \
   --start ${TOMORROW}T00:00:00+02:00 --duration PT24H \
-  --flex-context '{"consumption-price": {"sensor": 1}}' \
+  --flex-context '{"consumption-price": {"sensor": '"${FM_TOY_PRICE_SENSOR_ID}"'}}' \
   --flex-model '{"duration": "PT4H", "process-type": "SHIFTABLE", "power": 0.2, "time-restrictions": [{"start": "'"${TOMORROW}"'T15:00:00+02:00", "duration": "PT1H"}]}'
 
 echo "Now visit http://localhost:5000/assets/6/graphs to see all three schedules."