Adding Lrs yetanalytics as new stack + others fixes

docker-stacks/01-traefik/docker-compose.dev-lb.yml

@@ -31,7 +31,6 @@ services:
       SIMVA_TRAEFIK_SHLINK_KEY_FILENAME: ${SIMVA_TRAEFIK_SHLINK_KEY_FILENAME:-privkey.pem}
       #EXTERNAL SERVER LIST NAME
       SIMVA_EXTERNAL_DOMAIN: ${SIMVA_EXTERNAL_DOMAIN:-external.test}
-      SIMVA_SERVER_LIST_NAME: ${SIMVA_SERVER_LIST_NAME:-simva-servers}
       SIMVA_MINIO_HOST_SUBDOMAIN: ${SIMVA_MINIO_HOST_SUBDOMAIN:-minio}
       #INTERNAL USE HOST SUBDOMAINS
       SIMVA_INTERNAL_DOMAIN: ${SIMVA_INTERNAL_DOMAIN:-internal.test}
@@ -41,7 +40,8 @@ services:
       SIMVA_WHOAMI_NGINX_HOST_SUBDOMAIN: ${SIMVA_WHOAMI_NGINX_HOST_SUBDOMAIN:-nginx-whoami}
     volumes:
       - /etc/localtime:/etc/localtime:ro
-      - "${SIMVA_CONFIG_TEMPLATE_HOME:-/home/vagrant/docker-stacks/config-template}/traefik/nginx:/etc/nginx/templates:ro"
+      - "${SIMVA_CONFIG_TEMPLATE_HOME:-/home/vagrant/docker-stacks/config-template}/traefik/nginx/default.conf.template:/etc/nginx/templates/default.conf.template:ro"
+      - "${SIMVA_DATA_HOME:?Data home folder required}/traefik/nginx_upstream_hosts.txt:/etc/nginx/templates/nginx_upstream_hosts.txt:ro"
       - "${SIMVA_TLS_HOME:?TLS home folder required}:/etc/traefik/ssl:ro"
     depends_on:
       - traefik

docker-stacks/01-traefik/etc/hooks/before-start.d/00-check-rootCA.sh

@@ -33,7 +33,7 @@ if [[ -f "${SIMVA_ROOT_CA_FILE}" ]] && [[ "$rootCA_updated" == "false" ]]; then
             "$SIMVA_HOME/simva" backup "$CURRENT_STACK";
             "${HELPERS_STACK_HOME}/01-install-rootCA.sh"
             rootCA_updated=true;
-        else 
+        else
             echo "The root CA certificate is expired. Please update it before starting the stack."
             exit 1
         fi

docker-stacks/01-traefik/etc/hooks/before-start.d/20-nginx.sh

@@ -10,7 +10,7 @@ if [[ "${SIMVA_DEV_LOAD_BALANCER}" == "false" ]]; then
 fi
 OUTPUT_FILE="${SIMVA_DATA_HOME}/traefik/nginx_upstream_hosts.txt"
 mkdir -p "$(dirname "$OUTPUT_FILE")"
-echo "${SIMVA_EXTERNAL_DOMAIN};" > "$OUTPUT_FILE"
+echo "" > "$OUTPUT_FILE"
 for var in $(compgen -v | grep '^SIMVA_.*_HOST_SUBDOMAIN$'); do
   if [[ "$var" == *"NGINX"* ]] || [[ "$var" == *"MINIO_HOST"* ]] || [[ "$var" == *"SHLINK"* && $SIMVA_SHLINK_USE_SIMVA_EXTERNAL_DOMAIN == false ]]; then
     continue
@@ -21,4 +21,5 @@ for var in $(compgen -v | grep '^SIMVA_.*_HOST_SUBDOMAIN$'); do
     fi
   fi
 done
-export SIMVA_SERVER_LIST_NAME=$(cat "$OUTPUT_FILE");
+echo "Generated Nginx upstream hosts configuration:"
+cat "$OUTPUT_FILE"

docker-stacks/01-traefik/etc/hooks/helpers.d/02bis-install-shlink-wildcard-certificate.sh

@@ -3,24 +3,31 @@ set -euo pipefail
 [[ "${DEBUG:-false}" == "true" ]] && set -x
 
 if [[ $SIMVA_SHLINK_USE_SIMVA_EXTERNAL_DOMAIN  == "false" ]]; then 
-    if [[ "${SIMVA_TLS_GENERATE_SELF_SIGNED}" == "true" ]]; then
-        if [[ ! -e "${SIMVA_TRAEFIK_SHLINK_CERT_FILE}" ]]; then
-            mkcert \
-                -cert-file "${SIMVA_TRAEFIK_SHLINK_CERT_FILE}" \
-                -key-file "${SIMVA_TRAEFIK_SHLINK_KEY_FILE}" \
-                    "${SIMVA_SHLINK_EXTERNAL_DOMAIN}" \
-                    "localhost" \
-                    "127.0.0.1" \
-                    "${SIMVA_HOST_EXTERNAL_IP}"
-            chmod ${SIMVA_CERT_FILE_MOD} "${SIMVA_TRAEFIK_SHLINK_KEY_FILE}"
-            chmod ${SIMVA_CERT_FILE_MOD} "${SIMVA_TRAEFIK_SHLINK_CERT_FILE}"
-            source ${SIMVA_HOME}/bin/check-checksum.sh;
-            set +e
-            _check_checksum $SIMVA_TLS_HOME "${SIMVA_TRAEFIK_SHLINK_SHA256SUMS_FILE}" "${SIMVA_TRAEFIK_SHLINK_CERT_FILENAME}"
-            set -e
-        fi
+    if [[ -f "${SIMVA_TRAEFIK_SHLINK_CERT_FILE}" && -f "${SIMVA_TRAEFIK_SHLINK_KEY_FILE}" ]]; then
+        echo "Shlink cert file and key file already exist at '${SIMVA_TRAEFIK_SHLINK_CERT_FILE}' and '${SIMVA_TRAEFIK_SHLINK_KEY_FILE}' respectively. Skipping certificate generation."
     else
-        echo "Please insert your shlink cert file at '${SIMVA_TRAEFIK_SHLINK_CERT_FILE}' and shlink key file at '${SIMVA_TRAEFIK_SHLINK_KEY_FILE}' or run using SIMVA_TLS_GENERATE_SELF_SIGNED=true to self generate your certificates."
-        exit 1;
+        if [[ "${SIMVA_TLS_GENERATE_SELF_SIGNED}" == "true" ]]; then
+            if [[ ! -e "${SIMVA_TRAEFIK_SHLINK_CERT_FILE}" ]]; then
+                mkcert \
+                    -cert-file "${SIMVA_TRAEFIK_SHLINK_CERT_FILE}" \
+                    -key-file "${SIMVA_TRAEFIK_SHLINK_KEY_FILE}" \
+                        "${SIMVA_SHLINK_EXTERNAL_DOMAIN}" \
+                        "localhost" \
+                        "127.0.0.1" \
+                        "${SIMVA_HOST_EXTERNAL_IP}"
+                chmod ${SIMVA_CERT_FILE_MOD} "${SIMVA_TRAEFIK_SHLINK_KEY_FILE}"
+                chmod ${SIMVA_CERT_FILE_MOD} "${SIMVA_TRAEFIK_SHLINK_CERT_FILE}"
+                source ${SIMVA_HOME}/bin/check-checksum.sh;
+                set +e
+                _check_checksum $SIMVA_TLS_HOME "${SIMVA_TRAEFIK_SHLINK_SHA256SUMS_FILE}" "${SIMVA_TRAEFIK_SHLINK_CERT_FILENAME}"
+                set -e
+            else 
+                echo "Please insert your shlink cert file at '${SIMVA_TRAEFIK_SHLINK_CERT_FILE}' and shlink key file at '${SIMVA_TRAEFIK_SHLINK_KEY_FILE}' or run using SIMVA_TLS_GENERATE_SELF_SIGNED=true to self generate your certificates."
+                exit 1;
+            fi
+        else
+            echo "Please insert your shlink cert file at '${SIMVA_TRAEFIK_SHLINK_CERT_FILE}' and shlink key file at '${SIMVA_TRAEFIK_SHLINK_KEY_FILE}' or run using SIMVA_TLS_GENERATE_SELF_SIGNED=true to self generate your certificates."
+            exit 1;
+        fi
     fi
 fi

docker-stacks/02-keycloak/etc/hooks/before-migrate.d/10-remove-simva-realm-data.sh

@@ -7,7 +7,4 @@ ${SIMVA_BIN_HOME}/purge-folder-contents.sh \
 
 ${SIMVA_BIN_HOME}/purge-file-if-exist.sh \
     "${SIMVA_CONFIG_HOME}/keycloak/realm-data.dev.yml" \
-    "${SIMVA_CONFIG_HOME}/keycloak/realm-data.prod.yml"
-
-touch "${SIMVA_CONFIG_HOME}/keycloak/simva-realm-export/.exportinprogress"
-touch "${SIMVA_CONFIG_HOME}/keycloak/.migration"
+    "${SIMVA_CONFIG_HOME}/keycloak/realm-data.prod.yml"

docker-stacks/02-keycloak/etc/hooks/before-upgrade.d/upgrade.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+[[ "${DEBUG:-false}" == "true" ]] && set -x
+
+if [[ ${SIMVA_KEYCLOAK_VERSION%%.*} -gt 18 ]]; then
+    exit 0
+fi
+touch "${SIMVA_CONFIG_HOME}/keycloak/simva-realm-export/.exportinprogress"
+echo "$SIMVA_KEYCLOAK_VERSION" > "${SIMVA_CONFIG_HOME}/keycloak/.migration"

docker-stacks/02-keycloak/etc/hooks/helpers.d/keycloak-functions.sh

@@ -25,6 +25,9 @@ function __keycloak_login() {
     admin_user_file="$SIMVA_DATA_HOME/keycloak/.master_admin_user_created"
     echo "--- Checking Keycloak admin user status ---"
 
+    # Always use lowercase for admin username
+    admin_user_lc="${SIMVA_KEYCLOAK_ADMIN_USER,,}"
+
     if [[ -f "$admin_user_file" ]]; then
         echo "Admin user already persisted previously."
     else
@@ -40,13 +43,13 @@ function __keycloak_login() {
             set +e
             "${SIMVA_HOME}/bin/run-command.sh" /opt/keycloak/bin/kcadm.sh config credentials \
                 --server "https://${SIMVA_SSO_HOST_SUBDOMAIN}.${SIMVA_EXTERNAL_DOMAIN}" \
-                --realm "master" --user "${SIMVA_KEYCLOAK_ADMIN_USER}" \
+                --realm "master" --user "$admin_user_lc" \
                 --password "${SIMVA_KEYCLOAK_ADMIN_PASSWORD}"
             ret=$?
             set -e
             echo $ret
             if [[ $ret != 0 ]]; then
-                echo "Either temp admin '${SIMVA_KEYCLOAK_TMP_ADMIN_USER}' user and admin '${SIMVA_KEYCLOAK_ADMIN_USER}' user can't connect to Keycloak with the current password '${SIMVA_KEYCLOAK_ADMIN_PASSWORD}'."
+                echo "Either temp admin '${SIMVA_KEYCLOAK_TMP_ADMIN_USER}' user and admin '$admin_user_lc' user can't connect to Keycloak with the current password."
                 echo "Please update your simva-env.sh with the correct password configuration. Exiting..."
                 exit 1
             else 
@@ -55,30 +58,42 @@ function __keycloak_login() {
         else 
             echo "Checking if admin user exists in Keycloak..."
 
-            admin_user_present=$(
+            admin_user_present=$(\
                 "${SIMVA_HOME}/bin/run-command.sh" /opt/keycloak/bin/kcadm.sh get users \
-                    -r master -q username="${SIMVA_KEYCLOAK_ADMIN_USER}" -q exact=true --fields id,username 2>/dev/null
+                    -r master -q username="$admin_user_lc" -q exact=true --fields id,username 2>/dev/null
             )
-            if echo "$admin_user_present" | grep -q "\"username\" : \"${SIMVA_KEYCLOAK_ADMIN_USER}\""; then
+            if echo "$admin_user_present" | grep -iq "\"username\" *: *\"$admin_user_lc\""; then
                 echo "Admin exists"
                 echo "$admin_user_present"
                 echo "Admin user already exists in Keycloak."
             else
                 echo "Admin NOT found"
                 echo "Admin user doesn't exist. Creating through TMP admin login..."
                 "${SIMVA_HOME}/bin/run-command.sh" /opt/keycloak/bin/kcadm.sh create users \
-                    -r master -s username="${SIMVA_KEYCLOAK_ADMIN_USER}" -s enabled=true
+                    -r master -s username="$admin_user_lc" -s enabled=true
                 echo "Admin user created."
+            fi
+
+            # Check if the user already has the 'admin' role before adding
+            has_admin_role=$( \
+                "${SIMVA_HOME}/bin/run-command.sh" /opt/keycloak/bin/kcadm.sh get users \
+                    -r master -q username="$admin_user_lc" -q exact=true --fields id 2>/dev/null | \
+                jq -r '.[0].id' | \
+                xargs -I{} "${SIMVA_HOME}/bin/run-command.sh" /opt/keycloak/bin/kcadm.sh get users/{}/role-mappings/realm -r master 2>/dev/null | \
+                jq -r '.[] | select(.name=="admin") | .name' )
+            if [[ "$has_admin_role" != "admin" ]]; then
                 "${SIMVA_HOME}/bin/run-command.sh" /opt/keycloak/bin/kcadm.sh add-roles \
-                    -r master --uusername "${SIMVA_KEYCLOAK_ADMIN_USER}" --rolename admin
+                    -r master --uusername "$admin_user_lc" --rolename admin
+            else
+                echo "Admin user already has the 'admin' role."
             fi
-
+                    
             "${SIMVA_HOME}/bin/run-command.sh" /opt/keycloak/bin/kcadm.sh set-password \
-                -r master --username "${SIMVA_KEYCLOAK_ADMIN_USER}" \
+                -r master --username "$admin_user_lc" \
                 --new-password "${SIMVA_KEYCLOAK_ADMIN_PASSWORD}"
 
             # Delete the temporary admin user
-            tmp_user_json=$(
+            tmp_user_json=$(\
                 "${SIMVA_HOME}/bin/run-command.sh" /opt/keycloak/bin/kcadm.sh get users \
                     -r master -q username="${SIMVA_KEYCLOAK_TMP_ADMIN_USER}" -q exact=true --fields id 2>/dev/null
             )
@@ -97,11 +112,11 @@ function __keycloak_login() {
         touch "$admin_user_file"
     fi
 
-    # Now always login using the actual admin
+    # Now always login using the actual admin (lowercase)
     echo "--- Logging into Keycloak with the final admin ---"
     "${SIMVA_HOME}/bin/run-command.sh" /opt/keycloak/bin/kcadm.sh config credentials \
         --server "https://${SIMVA_SSO_HOST_SUBDOMAIN}.${SIMVA_EXTERNAL_DOMAIN}" \
-        --realm "master" --user "${SIMVA_KEYCLOAK_ADMIN_USER}" \
+        --realm "master" --user "$admin_user_lc" \
         --password "${SIMVA_KEYCLOAK_ADMIN_PASSWORD}"
 
     export KEYCLOAK_LOGIN_ON=true

docker-stacks/02-keycloak/etc/hooks/purge.d/01-remove-data-and-config.sh

@@ -17,7 +17,7 @@ ${SIMVA_BIN_HOME}/purge-file-if-exist.sh \
     "${SIMVA_DATA_HOME}/keycloak/.version" \
     "${SIMVA_DATA_HOME}/keycloak/.master_admin_user_created" \
     "${SIMVA_DATA_HOME}/keycloak/simva-realm-export\.exportinprogress" \
-    "${SIMVA_DATA_HOME}/keycloak/.migration"
+    "${SIMVA_CONFIG_HOME}/keycloak/.migration"
 
 "${SIMVA_BIN_HOME}/volumectl.sh" delete "kc_maria_db_data"
 "${SIMVA_BIN_HOME}/volumectl.sh" delete "kc_maria_db_backup_data"

docker-stacks/04-minio/etc/hooks/after-start.d/20-add-policy.sh

@@ -49,7 +49,9 @@ else
                 -c "mc config host add simva-minio ${minio_url} ${SIMVA_MINIO_ACCESS_KEY} ${SIMVA_MINIO_SECRET_KEY} ${extra_config} &&
                     mc ready simva-minio &&
                     $code &&
-                    mc --debug mb --ignore-existing simva-minio/${SIMVA_TRACES_BUCKET_NAME}"
+                    mc --debug mb --ignore-existing simva-minio/${SIMVA_TRACES_BUCKET_NAME}
+                    &&
+                    mc --debug mb --ignore-existing simva-minio/${SIMVA_BACKUP_BUCKET_NAME}"
         touch "${SIMVA_DATA_HOME}/minio/.minio-initialized";
     fi
 fi

docker-stacks/07-simva/MIGRATION_MONGO_SQLITE.MD

@@ -0,0 +1,2 @@
+To test the migration script to debug it run : 
+./simva uninstall 07-simva/ && rm -rf data/simva/sqlite_init && docker volume rm simva_sqlite_data && ./simva start 07-simva/

docker-stacks/07-simva/docker-compose.dev.yml

@@ -12,43 +12,40 @@ x-default-opts:
 
 services:
   simva-api:
-    build: ${SIMVA_API_GIT_REPO:-/home/vagrant/docker-stacks/data/simva/simva-api}
+    build: ${SIMVA_API_GIT_REPO:-/home/vagrant/simva}
     restart: unless-stopped
     environment:
       LOG_LEVEL: "debug"
+      JSTRACKER_LOCAL_DEPLOYMENT: "${SIMVA_JSTRACKER_LIBRARY_LOCAL_DEPLOYMENT}"
+      JSTRACKER_LOCAL_PATH: "/home/node/js-tracker"
+      JSTRACKER_ENTRYPOINT_SCRIPT: "/home/node/entrypoint.d/setup-local-deps.js"
     volumes:
-      - ${SIMVA_API_GIT_REPO:-/home/vagrant/docker-stacks/data/simva/simva-api}/src:/home/node/app/src
-      - ${SIMVA_API_GIT_REPO:-/home/vagrant/docker-stacks/data/simva/simva-api}/tests:/home/node/app/tests
-      - ${SIMVA_API_GIT_REPO:-/home/vagrant/docker-stacks/data/simva/simva-api}/public:/home/node/app/public
-      - ${SIMVA_API_GIT_REPO:-/home/vagrant/docker-stacks/data/simva/simva-api}/package.json:/home/node/app/package.json
-      - ${SIMVA_API_GIT_REPO:-/home/vagrant/docker-stacks/data/simva/simva-api}/package-lock.json:/home/node/app/package-lock.json
-      - ${SIMVA_API_GIT_REPO:-/home/vagrant/docker-stacks/data/simva/simva-api}/jest.config.cjs:/home/node/app/jest.config.cjs
-      - ${SIMVA_API_GIT_REPO:-/home/vagrant/docker-stacks/data/simva/simva-api}/tsconfig.jest.json:/home/node/app/tsconfig.jest.json
-      - ${SIMVA_API_GIT_REPO:-/home/vagrant/docker-stacks/data/simva/simva-api}/tsconfig.json:/home/node/app/tsconfig.json
-      - ${SIMVA_API_GIT_REPO:-/home/vagrant/docker-stacks/data/simva/simva-api}/coverage:/home/node/app/coverage
-      - ${SIMVA_API_GIT_REPO:-/home/vagrant/docker-stacks/data/simva/simva-api}:/home/node/app/
+      - ${SIMVA_API_GIT_REPO:-/home/vagrant/simva}:/home/node/app/
+      - ${SIMVA_JSTRACKER_LIBRARY_LOCAL_PATH:-/home/vagrant/js-tracker}:/home/node/js-tracker
+      - /home/node/js-tracker/node_modules  # Exclude node_modules
       - /home/node/app/node_modules  # Exclude node_modules
+      - ${STACK_HOME?STACK home folder required}/etc/entrypoint.d/setup-local-deps.js:/home/node/entrypoint.d/setup-local-deps.js
     ports:
       - "127.0.0.1:9229:9229"
       - "127.0.0.1:9232:9230"
   simva-front:
-    build: ${SIMVA_FRONT_GIT_REPO:-/home/vagrant/docker-stacks/data/simva/simva-front}
+    build: ${SIMVA_FRONT_GIT_REPO:-/home/vagrant/simva-front}
     restart: unless-stopped
     environment:
       LOG_LEVEL: "debug"
     volumes:
-      - ${SIMVA_FRONT_GIT_REPO:-/home/vagrant/docker-stacks/data/simva/simva-front}:/home/node/app/
+      - ${SIMVA_FRONT_GIT_REPO:-/home/vagrant/simva-front}:/home/node/app/
       - /home/node/app/node_modules  # Exclude node_modules
     ports:
       - "127.0.0.1:9230:9229"
       - "127.0.0.1:9233:9230"
   simva-trace-allocator:
-    build: ${SIMVA_TRACE_ALLOCATOR_GIT_REPO:-/home/vagrant/docker-stacks/data/simva/simva-trace-allocator}
+    build: ${SIMVA_TRACE_ALLOCATOR_GIT_REPO:-/home/vagrant/simva-trace-allocator}
     restart: unless-stopped
     environment:
       LOG_LEVEL: "debug"
     volumes:
-      - ${SIMVA_TRACE_ALLOCATOR_GIT_REPO:-/home/vagrant/docker-stacks/data/simva/simva-trace-allocator}:/home/node/app/
+      - ${SIMVA_TRACE_ALLOCATOR_GIT_REPO:-/home/vagrant/simva-trace-allocator}:/home/node/app/
       - /home/node/app/node_modules  # Exclude node_modules
     ports:
       - "127.0.0.1:9231:9229"

docker-stacks/07-simva/docker-compose.yml

@@ -115,6 +115,9 @@ services:
       SHLINK_PORT: ${SIMVA_SHLINK_PORT:-443}
       SHLINK_PROTOCOL: ${SIMVA_SHLINK_EXTERNAL_PROTOCOL:-https}
       SHLINK_SERVER_API_KEY: ${SIMVA_SHLINK_API_KEY:-password}
+      LRS_ENDPOINT: "https://${SIMVA_LRS_HOST_SUBDOMAIN:-lrs}.${SIMVA_EXTERNAL_DOMAIN:-external.test}/"
+      LRS_API_KEY_DEFAULT: "${SIMVA_LRS_API_KEY_DEFAULT:-my_api_key}"
+      LRS_API_SECRET_DEFAULT: "${SIMVA_LRS_API_SECRET_DEFAULT:-my_api_secret}"
     volumes:
       - /etc/localtime:/etc/localtime:ro
       - ${SIMVA_TLS_HOME?TLS home folder required}:/var/lib/simva/tls

docker-stacks/07-simva/etc/entrypoint.d/docker-startup-api.sh

@@ -7,6 +7,8 @@ cd /home/node/app
 #start api
 echo "${NODE_ENV}"
 if [[ "${NODE_ENV}" == "development" ]]; then
+  # Setup local dependencies if needed (js-tracker)
+  node ${JSTRACKER_ENTRYPOINT_SCRIPT:-/home/node/entrypoint.d/setup-local-deps.js}
   if [[ "${ENABLE_DEBUG_PROFILING:-false}" == "true" ]]; then
       if [[ ! -e ${PROFILING_FOLDER} ]]; then 
         mkdir -p ${PROFILING_FOLDER}