Call enhancer, carpool events

.gitignore

@@ -132,6 +132,7 @@ setup.ipynb
 
 *~
 /.idea/
+/.vscode/
 secrets
 gtfs/*.zip
 gtfs/*.pbf
@@ -142,4 +143,12 @@ data/failed/
 data/trash/
 data/gtfs/
 data/tmp
-
+data/users/**
+data/**
+
+#these files are under app/static but they get copied to the outside directory on startup
+logging.conf
+config
+static/**
+templates/**
+conf/**

Dockerfile

@@ -15,27 +15,29 @@ RUN \
 	# Remove package index obtained by `apt update`.
 	&& rm -rf /var/lib/apt/lists/*
 
-ENV ADMIN_TOKEN=''
-ENV RIDE2GO_TOKEN=''
+ ENV ADMIN_TOKEN=''
+ ENV RIDE2GO_TOKEN=''
+ ENV SECRET_KEY=''
 
 EXPOSE 80
 
-ARG PLUGINS
-
 COPY requirements.txt /app/requirements.txt
-RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt ${PLUGINS}
+RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt
 
 # set MODULE_NAME explicitly
 ENV MODULE_NAME=amarillo.main
 
 COPY ./amarillo /app/amarillo
-COPY enhancer.py /app
-COPY prestart.sh /app
-COPY ./static /app/static
-COPY ./templates /app/templates
-COPY config /app
-COPY logging.conf /app
-COPY ./conf /app/conf
+COPY ./amarillo/plugins /app/amarillo/plugins
+COPY ./amarillo/static/static /app/static
+COPY ./amarillo/static/templates /app/templates
+COPY ./amarillo/static/config /app
+COPY ./amarillo/static/logging.conf /app
+COPY ./amarillo/static/data /app/data
+
+# Create the error.log, otherwise we get a permission error when we try to write to it
+RUN touch /app/error.log
+RUN chmod 777 /app/error.log
 
 # This image inherits uvicorn-gunicorn's CMD. If you'd like to start uvicorn, use this instead
 # CMD ["uvicorn", "amarillo.main:app", "--host", "0.0.0.0", "--port", "8000"]

Jenkinsfile

@@ -1,17 +1,17 @@
 pipeline {
-    agent any
+    agent { label 'builtin' }
     environment {
         GITEA_CREDS = credentials('AMARILLO-JENKINS-GITEA-USER')
-        PYPI_CREDS = credentials('AMARILLO-JENKINS-PYPI-USER')
         TWINE_REPO_URL = "https://git.gerhardt.io/api/packages/amarillo/pypi"
         PLUGINS_REPO_URL = "git.gerhardt.io/api/packages/amarillo/pypi/simple"
-        DOCKER_REGISTRY_URL = 'https://git.gerhardt.io'
+        DOCKER_REGISTRY = 'git.gerhardt.io'
+        DERIVED_DOCKERFILE = 'standard.Dockerfile'
         OWNER = 'amarillo'
+        BASE_IMAGE_NAME = 'amarillo-base'
         IMAGE_NAME = 'amarillo'
-        AMARILLO_DISTRIBUTION = '0.2'
-        TAG = "${AMARILLO_DISTRIBUTION}.${BUILD_NUMBER}"
-        PLUGINS = 'amarillo-metrics amarillo-enhancer amarillo-grfs-export'
-        DEPLOY_WEBHOOK_URL = 'http://amarillo.mfdz.de:8888/mitanand'
+        AMARILLO_DISTRIBUTION = '0.3'
+        TAG = "${AMARILLO_DISTRIBUTION}.${BUILD_NUMBER}${env.BRANCH_NAME == 'main' ? '' : '-' + env.BRANCH_NAME}"
+        DEPLOY_WEBHOOK_URL = "http://amarillo.mfdz.de:8888/${env.BRANCH_NAME}" 
         DEPLOY_SECRET = credentials('AMARILLO-JENKINS-DEPLOY-SECRET')
     }
     stages {
@@ -42,35 +42,56 @@ pipeline {
                 sh 'python3 -m twine upload --skip-existing --verbose --repository-url $TWINE_REPO_URL --username $GITEA_CREDS_USR --password $GITEA_CREDS_PSW ./dist/*'              
             }
         }
-        stage('Publish package to PyPI') {
+        stage('Build base docker image') {
             when {
-                branch 'release'
+                isDeployBranch()
             }
             steps {
-                sh 'python3 -m twine upload --verbose --username $PYPI_CREDS_USR --password $PYPI_CREDS_PSW ./dist/*'              
+                echo 'Building image'
+                script {
+                    docker.build("${OWNER}/${BASE_IMAGE_NAME}:${TAG}")
+                }
+            }
+        }
+        stage('Push base image to container registry') {
+            when {
+                isDeployBranch()
+            }
+            steps {
+                echo 'Pushing image to registry'
+                script {
+                    docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){
+                        def image = docker.image("${OWNER}/${BASE_IMAGE_NAME}:${TAG}")
+                        image.push()
+                        image.push('latest')
+                    }
+                }
             }
         }
-        stage('Build Mitanand docker image') {
+        stage('Build derived docker image') {
             when {
-                branch 'mitanand'
+                isDeployBranch()
             }
             steps {
                 echo 'Building image'
                 script {
-                    docker.build("${OWNER}/${IMAGE_NAME}:${TAG}",
-                    //--no-cache to make sure plugins are updated
-                     "--no-cache --build-arg='PACKAGE_REGISTRY_URL=${PLUGINS_REPO_URL}' --build-arg='PLUGINS=${PLUGINS}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .")
+                    docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){
+                        docker.build("${OWNER}/${IMAGE_NAME}:${TAG}",
+                        //--no-cache to make sure plugins are updated
+                        "-f ${DERIVED_DOCKERFILE} --no-cache --build-arg='PACKAGE_REGISTRY_URL=${PLUGINS_REPO_URL}' --build-arg='DOCKER_REGISTRY=${DOCKER_REGISTRY}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .")
+                    }
+
                 }
             }
         }
-        stage('Push image to container registry') {
+        stage('Push derived image to container registry') {
             when {
-                branch 'mitanand'
+                isDeployBranch()
             }
             steps {
                 echo 'Pushing image to registry'
                 script {
-                    docker.withRegistry(DOCKER_REGISTRY_URL, 'AMARILLO-JENKINS-GITEA-USER'){
+                    docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){
                         def image = docker.image("${OWNER}/${IMAGE_NAME}:${TAG}")
                         image.push()
                         image.push('latest')
@@ -80,7 +101,7 @@ pipeline {
         }
         stage('Notify CD script') {
             when {
-                branch 'mitanand'
+                isDeployBranch()
             }
             steps {
                 echo 'Triggering deploy webhook'
@@ -93,3 +114,7 @@ pipeline {
         }
     }
 }
+
+def isDeployBranch() {
+    return anyOf { branch 'main'; branch 'dev'; branch 'mitanand' }
+}

MANIFEST.in

@@ -0,0 +1,2 @@
+recursive-include amarillo/static/ *
+recursive-include amarillo/tests/ *

README.md

@@ -9,9 +9,7 @@ An Amarillo is a [yellow-dressed person](https://www.dreamstime.com/sancti-spiri
 - Python 3.9.2 with pip
 - python3-venv
 
-Create a virtual environment `python3 -m venv venv`.
-
-Activate the environment and install the dependencies `pip install -r requirements.txt`.
+Create a virtual environment `python3 -m venv venv`. Activate the environment with `source venv/bin/activate` and install the dependencies `pip install -r requirements.txt`.
 
 Run `uvicorn amarillo.main:app`. 
 
@@ -22,23 +20,25 @@ In development, you can use `--reload`.
 - `env`
 - `ADMIN_TOKEN`
 
+E.g. set the environment variable like this: `export ADMIN_TOKEN=YOUR_SECRET_TOKEN_HERE`.
+
 ## Security
 
 All endpoints are protected by an API-Key in the HTTP header. 
 There is a special *admin* user. 
 For this user, the API-Key must be passed in as an environment variable when 
 Amarillo is started.
 
-The admin can create additional API-Keys in the `/agencyconf` endpoint. This 
+The admin can create additional API-Keys in the `/users` endpoint. This
 endpoint is always available but not always shown in `/docs`, especially not
 when running in production. 
-The Swagger docs for `/agencyconf` can be seen on the MFDZ demo server. 
+The Swagger docs for `/users` can be seen on the MFDZ demo server.
 
 Permissions work this way
 - the admin is allowed to call all operations on all resources. Only the admin
-  can create new API-Keys by POSTing an `AgencyConf` JSON object to `/agencyconf`. 
+  can create new API-Keys by POSTing an `users` JSON object to `/users`.
 - API-Keys for agencies are allowed to POST/PUT/GET/DELETE their own 
-  resources and GET some public resources.  
+  resources and GET some public resources.
 
 ## Development
 

amarillo/configuration.py

@@ -1,21 +1,14 @@
 # separate file so that it can be imported without initializing FastAPI
 from amarillo.utils.container import container
-import json
 import logging
-from glob import glob
 
-from amarillo.models.Carpool import Agency, Carpool, Region
-from amarillo.services import stops
-from amarillo.services import trips
-from amarillo.services.agencyconf import AgencyConfService, agency_conf_directory
-from amarillo.services.carpools import CarpoolService
+from amarillo.services.users import UserService, user_conf_directory
 from amarillo.services.agencies import AgencyService
 from amarillo.services.regions import RegionService
 
 from amarillo.services.config import config
 
 from amarillo.utils.utils import assert_folder_exists
-import amarillo.services.gtfs_generator as gtfs_generator
 
 logger = logging.getLogger(__name__)
 
@@ -34,12 +27,12 @@ def create_required_directories():
             assert_folder_exists(f'data/{subdir}/{agency_id}')
 
     # Agency configurations
-    assert_folder_exists(agency_conf_directory)
+    assert_folder_exists(user_conf_directory)
 
 
 def configure_services():
-    container['agencyconf'] = AgencyConfService()
-    logger.info("Loaded %d agency configuration(s)", len(container['agencyconf'].agency_id_to_agency_conf))
+    container['users'] = UserService()
+    logger.info("Loaded %d user configuration(s)", len(container['users'].user_id_to_user_conf))
 
     container['agencies'] = AgencyService()
     logger.info("Loaded %d agencies", len(container['agencies'].agencies))
@@ -49,42 +42,6 @@ def configure_services():
 
     create_required_directories()
 
-
-def configure_enhancer_services():
-    configure_services()
-
-    logger.info("Load stops...")
-    with open(config.stop_sources_file) as stop_sources_file:
-        stop_sources = json.load(stop_sources_file)
-        stop_store = stops.StopsStore(stop_sources)
-
-    stop_store.load_stop_sources()
-    container['stops_store'] = stop_store
-    container['trips_store'] = trips.TripStore(stop_store)
-    container['carpools'] = CarpoolService(container['trips_store'])
-
-    logger.info("Restore carpools...")
-
-    for agency_id in container['agencies'].agencies:
-        for carpool_file_name in glob(f'data/carpool/{agency_id}/*.json'):
-            try:
-                with open(carpool_file_name) as carpool_file:
-                    carpool = Carpool(**(json.load(carpool_file)))
-                    container['carpools'].put(carpool.agency, carpool.id, carpool)
-            except Exception as e:
-                logger.warning("Issue during restore of carpool %s: %s", carpool_file_name, repr(e))
-
-        # notify carpool about carpools in trash, as delete notifications must be sent
-        for carpool_file_name in glob(f'data/trash/{agency_id}/*.json'):
-            with open(carpool_file_name) as carpool_file:
-                carpool = Carpool(**(json.load(carpool_file)))
-                container['carpools'].delete(carpool.agency, carpool.id)
-
-    logger.info("Restored carpools: %s", container['carpools'].get_all_ids())
-    logger.info("Starting scheduler")
-    gtfs_generator.start_schedule()
-
-
 def configure_admin_token():
     if config.admin_token is None:
         message = "ADMIN_TOKEN environment variable not set"

amarillo/main.py

@@ -5,10 +5,15 @@
 import mimetypes
 from starlette.staticfiles import StaticFiles
 
+from amarillo.utils.utils import copy_static_files
+#this has to run before app.configuration is imported, otherwise we get validation error for config because the config file is not copied yet
+copy_static_files(["data", "static", "templates", "logging.conf", "config"]) 
+
 import amarillo.plugins
 from amarillo.services.config import config
 from amarillo.configuration import configure_services, configure_admin_token
-from amarillo.routers import carpool, agency, agencyconf, region
+from amarillo.routers import carpool, agency, users, region
+import amarillo.services.oauth2 as oauth2
 from fastapi import FastAPI
 
 # https://pydantic-docs.helpmanual.io/usage/settings/
@@ -76,8 +81,9 @@
 
 app.include_router(carpool.router)
 app.include_router(agency.router)
-app.include_router(agencyconf.router)
+app.include_router(users.router)
 app.include_router(region.router)
+app.include_router(oauth2.router)
 
 
 def iter_namespace(ns_pkg):