diff --git a/.gitignore b/.gitignore index 83461d6..e3f2885 100644 --- a/.gitignore +++ b/.gitignore @@ -132,6 +132,7 @@ setup.ipynb *~ /.idea/ +/.vscode/ secrets gtfs/*.zip gtfs/*.pbf @@ -142,4 +143,11 @@ data/failed/ data/trash/ data/gtfs/ data/tmp - +data/** + +#these files are under app/static but they get copied to the outside directory on startup +logging.conf +config +static/** +templates/** +conf/** diff --git a/Dockerfile b/Dockerfile index 9575a5c..2cb8f63 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,17 +4,6 @@ LABEL maintainer="info@mfdz.de" WORKDIR /app -RUN \ - apt update \ - && apt install -y \ - # GDAL headers are required for fiona, which is required for geopandas. - # Also gcc is used to compile C++ code. - libgdal-dev g++ \ - # libspatialindex is required for rtree. - libspatialindex-dev \ - # Remove package index obtained by `apt update`. - && rm -rf /var/lib/apt/lists/* - ENV ADMIN_TOKEN='' ENV RIDE2GO_TOKEN='' @@ -27,13 +16,12 @@ RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt ENV MODULE_NAME=amarillo.main COPY ./amarillo /app/amarillo -COPY enhancer.py /app -COPY prestart.sh /app -COPY ./static /app/static -COPY ./templates /app/templates -COPY config /app -COPY logging.conf /app -COPY ./conf /app/conf +COPY ./amarillo/plugins /app/amarillo/plugins +COPY ./amarillo/static/static /app/static +COPY ./amarillo/static/templates /app/templates +COPY ./amarillo/static/config /app +COPY ./amarillo/static/logging.conf /app +COPY ./amarillo/static/data /app/data # This image inherits uvicorn-gunicorn's CMD. If you'd like to start uvicorn, use this instead # CMD ["uvicorn", "amarillo.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/Jenkinsfile b/Jenkinsfile index 2715959..062a50b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,17 +1,21 @@ pipeline { - agent any + agent { label 'builtin' } environment { GITEA_CREDS = credentials('AMARILLO-JENKINS-GITEA-USER') PYPI_CREDS = credentials('AMARILLO-JENKINS-PYPI-USER') TWINE_REPO_URL = "https://git.gerhardt.io/api/packages/amarillo/pypi" - PLUGINS_REPO_URL = "git.gerhardt.io/api/packages/amarillo/pypi/simple" - DOCKER_REGISTRY_URL = 'https://git.gerhardt.io' + PLUGINS_REPO_URL = "https://git.gerhardt.io/api/packages/amarillo/pypi/simple" + PYPI_REPO_URL = "https://pypi.org/simple" + DOCKER_REGISTRY = 'git.gerhardt.io' + DERIVED_DOCKERFILE = 'standard.Dockerfile' + MITANAND_DOCKERFILE = 'mitanand.Dockerfile' OWNER = 'amarillo' + BASE_IMAGE_NAME = 'amarillo-base' IMAGE_NAME = 'amarillo' - AMARILLO_DISTRIBUTION = '0.2' - TAG = "${AMARILLO_DISTRIBUTION}.${BUILD_NUMBER}" - PLUGINS = 'amarillo-metrics amarillo-enhancer amarillo-grfs-export' - DEPLOY_WEBHOOK_URL = 'http://amarillo.mfdz.de:8888/mitanand' + MITANAND_IMAGE_NAME = 'amarillo-mitanand' + AMARILLO_DISTRIBUTION = '2.0.0' + TAG = "${AMARILLO_DISTRIBUTION}${env.BRANCH_NAME == 'main' ? '' : '-' + env.BRANCH_NAME}-${BUILD_NUMBER}" + DEPLOY_WEBHOOK_URL = "http://amarillo.mfdz.de:8888/dev" DEPLOY_SECRET = credentials('AMARILLO-JENKINS-DEPLOY-SECRET') } stages { @@ -44,33 +48,50 @@ pipeline { } stage('Publish package to PyPI') { when { - branch 'release' + branch 'main' } steps { sh 'python3 -m twine upload --verbose --username $PYPI_CREDS_USR --password $PYPI_CREDS_PSW ./dist/*' } } - stage('Build Mitanand docker image') { - when { - branch 'mitanand' - } + stage('Build base docker image') { steps { echo 'Building image' script { - docker.build("${OWNER}/${IMAGE_NAME}:${TAG}", - //--no-cache to make sure plugins are updated - "--no-cache --build-arg='PACKAGE_REGISTRY_URL=${PLUGINS_REPO_URL}' --build-arg='PLUGINS=${PLUGINS}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .") + docker.build("${OWNER}/${BASE_IMAGE_NAME}:${TAG}") } } } - stage('Push image to container registry') { - when { - branch 'mitanand' + stage('Push base image to container registry') { + steps { + echo 'Pushing image to registry' + script { + docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){ + def image = docker.image("${OWNER}/${BASE_IMAGE_NAME}:${TAG}") + image.push() + image.push('latest') + } + } } + } + stage('Build derived docker image') { + steps { + echo 'Building image' + script { + docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){ + docker.build("${OWNER}/${IMAGE_NAME}:${TAG}", + //--no-cache to make sure plugins are updated + "-f ${DERIVED_DOCKERFILE} --no-cache --build-arg='PACKAGE_REGISTRY_URL=${env.BRANCH_NAME == 'main' ? env.PYPI_REPO_URL : env.PLUGINS_REPO_URL}' --build-arg='DOCKER_REGISTRY=${DOCKER_REGISTRY}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .") + } + + } + } + } + stage('Push derived image to container registry') { steps { echo 'Pushing image to registry' script { - docker.withRegistry(DOCKER_REGISTRY_URL, 'AMARILLO-JENKINS-GITEA-USER'){ + docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){ def image = docker.image("${OWNER}/${IMAGE_NAME}:${TAG}") image.push() image.push('latest') @@ -78,18 +99,46 @@ pipeline { } } } - stage('Notify CD script') { - when { - branch 'mitanand' + + stage('Build mitanand docker image') { + steps { + echo 'Building image' + script { + docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){ + docker.build("${OWNER}/${MITANAND_IMAGE_NAME}:${TAG}", + //--no-cache to make sure plugins are updated + "-f ${MITANAND_DOCKERFILE} --no-cache --build-arg='PACKAGE_REGISTRY_URL=${env.BRANCH_NAME == 'main' ? env.PYPI_REPO_URL : env.PLUGINS_REPO_URL}' --build-arg='DOCKER_REGISTRY=${DOCKER_REGISTRY}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .") + } + + } } + } + stage('Push mitanand image to container registry') { steps { - echo 'Triggering deploy webhook' + echo 'Pushing image to registry' script { - def response = httpRequest contentType: 'APPLICATION_JSON', - httpMode: 'POST', requestBody: '{}', authentication: 'AMARILLO-JENKINS-DEPLOY-SECRET', - url: "${DEPLOY_WEBHOOK_URL}" + docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){ + def image = docker.image("${OWNER}/${MITANAND_IMAGE_NAME}:${TAG}") + image.push() + image.push('latest') + } } } } + // stage('Notify CD script') { + // when { + // not { + // branch 'main' + // } + // } + // steps { + // echo 'Triggering deploy webhook' + // script { + // def response = httpRequest contentType: 'APPLICATION_JSON', + // httpMode: 'POST', requestBody: '{}', authentication: 'AMARILLO-JENKINS-DEPLOY-SECRET', + // url: "${DEPLOY_WEBHOOK_URL}" + // } + // } + // } } -} +} \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..0daae06 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include amarillo/static/ * +recursive-include amarillo/tests/ * \ No newline at end of file diff --git a/README.md b/README.md index d78bad7..a2dd39f 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,17 @@ An Amarillo is a [yellow-dressed person](https://www.dreamstime.com/sancti-spiri ## Setup -- Python 3.9.2 with pip +- Python 3.10 with pip - python3-venv -Create a virtual environment `python3 -m venv venv`. +Create a virtual environment: +`python3 -m venv venv`. + +Activate the environment: +`. venv/bin/activate` + +Install the dependencies: `pip install -r requirements.txt`. -Activate the environment and install the dependencies `pip install -r requirements.txt`. Run `uvicorn amarillo.main:app`. @@ -54,13 +59,17 @@ $ protoc --proto_path=. --python_out=../services/gtfsrt gtfs-realtime.proto real $ sed 's/import gtfs_realtime_pb2/import amarillo.services.gtfsrt.gtfs_realtime_pb2/g' ../services/gtfsrt/realtime_extension_pb2.py | sponge ../services/gtfsrt/realtime_extension_pb2.py ``` +### Develop Amarillo and its plugins together + +To the develop Amarillo and its plugins concurrently, clone this repo and all of the repos you would like to make changes to. Install the local version of the plugin(s) into the virtual environment, simply using `pip install /path/to/plugin`, or `pip install -e /path/to/plugin` if your environment supports editable installs. + ## Testing In the top directory, run `pytest amarillo/tests`. ## Docker -Based on [tiangolo/uvicorn-gunicorn:python3.9-slim](https://github.com/tiangolo/uvicorn-gunicorn-docker) +Based on [tiangolo/uvicorn-gunicorn:python3.10-slim](https://github.com/tiangolo/uvicorn-gunicorn-docker) - build `docker build -t amarillo .` - run `docker run --rm --name amarillo -p 8000:80 -e MAX_WORKERS="1" -e ADMIN_TOKEN=$ADMIN_TOKEN -e RIDE2GO_TOKEN=$RIDE2GO_TOKEN -e TZ=Europe/Berlin -v $(pwd)/data:/app/data amarillo` diff --git a/amarillo/configuration.py b/amarillo/configuration.py index 669c988..7c42196 100644 --- a/amarillo/configuration.py +++ b/amarillo/configuration.py @@ -1,21 +1,15 @@ # separate file so that it can be imported without initializing FastAPI +from amarillo.routers.carpool import enhance_missing_carpools from amarillo.utils.container import container -import json import logging -from glob import glob -from amarillo.models.Carpool import Agency, Carpool, Region -from amarillo.services import stops -from amarillo.services import trips from amarillo.services.agencyconf import AgencyConfService, agency_conf_directory -from amarillo.services.carpools import CarpoolService from amarillo.services.agencies import AgencyService from amarillo.services.regions import RegionService from amarillo.services.config import config from amarillo.utils.utils import assert_folder_exists -import amarillo.services.gtfs_generator as gtfs_generator logger = logging.getLogger(__name__) @@ -48,42 +42,7 @@ def configure_services(): logger.info("Loaded %d regions", len(container['regions'].regions)) create_required_directories() - - -def configure_enhancer_services(): - configure_services() - - logger.info("Load stops...") - with open(config.stop_sources_file) as stop_sources_file: - stop_sources = json.load(stop_sources_file) - stop_store = stops.StopsStore(stop_sources) - - stop_store.load_stop_sources() - container['stops_store'] = stop_store - container['trips_store'] = trips.TripStore(stop_store) - container['carpools'] = CarpoolService(container['trips_store']) - - logger.info("Restore carpools...") - - for agency_id in container['agencies'].agencies: - for carpool_file_name in glob(f'data/carpool/{agency_id}/*.json'): - try: - with open(carpool_file_name) as carpool_file: - carpool = Carpool(**(json.load(carpool_file))) - container['carpools'].put(carpool.agency, carpool.id, carpool) - except Exception as e: - logger.warning("Issue during restore of carpool %s: %s", carpool_file_name, repr(e)) - - # notify carpool about carpools in trash, as delete notifications must be sent - for carpool_file_name in glob(f'data/trash/{agency_id}/*.json'): - with open(carpool_file_name) as carpool_file: - carpool = Carpool(**(json.load(carpool_file))) - container['carpools'].delete(carpool.agency, carpool.id) - - logger.info("Restored carpools: %s", container['carpools'].get_all_ids()) - logger.info("Starting scheduler") - gtfs_generator.start_schedule() - + enhance_missing_carpools() def configure_admin_token(): if config.admin_token is None: diff --git a/amarillo/main.py b/amarillo/main.py index fc1ef12..1d3c34e 100644 --- a/amarillo/main.py +++ b/amarillo/main.py @@ -1,21 +1,26 @@ import logging.config - -from amarillo.configuration import configure_services, configure_admin_token -from amarillo.services.config import config - -logging.config.fileConfig('logging.conf', disable_existing_loggers=False) -logger = logging.getLogger("main") - +import importlib +import pkgutil import uvicorn import mimetypes from starlette.staticfiles import StaticFiles +from amarillo.utils.utils import copy_static_files +#this has to run before app.configuration is imported, otherwise we get validation error for config because the config file is not copied yet +copy_static_files(["data", "static", "templates", "logging.conf", "config"]) + +import amarillo.plugins +from amarillo.services.config import config +from amarillo.configuration import configure_services, configure_admin_token from amarillo.routers import carpool, agency, agencyconf, region from fastapi import FastAPI # https://pydantic-docs.helpmanual.io/usage/settings/ from amarillo.views import home +logging.config.fileConfig('logging.conf', disable_existing_loggers=False) +logger = logging.getLogger("main") + logger.info("Hello Amarillo!") app = FastAPI(title="Amarillo - The Carpooling Intermediary", @@ -65,6 +70,10 @@ "description": "Demo server by MFDZ", "url": "https://amarillo.mfdz.de" }, + { + "description": "Mitanand Amarillo service", + "url": "https://mitanand.mfdz.de" + }, { "description": "Dev server for development", "url": "https://amarillo-dev.mfdz.de" @@ -79,10 +88,30 @@ app.include_router(region.router) +def iter_namespace(ns_pkg): + # Source: https://packaging.python.org/guides/creating-and-discovering-plugins/ + return pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".") + +def load_plugins(): + discovered_plugins = { + name: importlib.import_module(name) + for finder, name, ispkg + in iter_namespace(amarillo.plugins) + } + logger.info(f"Discovered plugins: {list(discovered_plugins.keys())}") + + for name, module in discovered_plugins.items(): + if hasattr(module, "setup"): + logger.info(f"Running setup function for {name}") + module.setup(app) + + else: logger.info(f"Did not find setup function for {name}") + def configure(): configure_admin_token() configure_services() configure_routing() + load_plugins() def configure_routing(): diff --git a/amarillo/models/Carpool.py b/amarillo/models/Carpool.py index ffbb995..60bfd6d 100644 --- a/amarillo/models/Carpool.py +++ b/amarillo/models/Carpool.py @@ -4,7 +4,7 @@ from datetime import time from pydantic import BaseModel, Field from geojson_pydantic.geometries import LineString -from enum import Enum +from enum import Enum, IntEnum NumType = Union[float, int] @@ -24,6 +24,15 @@ class PickupDropoffType(str, Enum): only_pickup = "only_pickup" only_dropoff = "only_dropoff" +class YesNoEnum(IntEnum): + yes = 1 + no = 2 + +class LuggageSize(IntEnum): + small = 1 + medium = 2 + large = 3 + class StopTime(BaseModel): id: Optional[str] = Field( None, @@ -111,7 +120,83 @@ class Region(BaseModel): bbox: Tuple[NumType, NumType, NumType, NumType] = Field( description="Bounding box of the region. Format is [minLon, minLat, maxLon, maxLat]", examples=[[10.5,49.2,11.3,51.3]]) + +class RidesharingInfo(BaseModel): + number_free_seats: int = Field( + description="Number of free seats", + ge=0, + examples=[3]) + + same_gender: Optional[YesNoEnum] = Field( + None, + description="Trip only for same gender:" + "1: Yes" + "2: No", + examples=[1]) + luggage_size: Optional[LuggageSize] = Field( + None, + description="Size of the luggage:" + "1: small" + "2: medium" + "3: large", + examples=[3]) + animal_car: Optional[YesNoEnum] = Field( + None, + description="Animals in Car allowed:" + "1: Yes" + "2: No", + examples=[2]) + + car_model: Optional[str] = Field( + None, + description="Car model", + min_length=1, + max_length=48, + examples=["Golf"]) + car_brand: Optional[str] = Field( + None, + description="Car brand", + min_length=1, + max_length=48, + examples=["VW"]) + + creation_date: datetime = Field( + description="Date when trip was created", + examples=["2022-02-13T20:20:39+00:00"]) + + smoking: Optional[YesNoEnum] = Field( + None, + description="Smoking allowed:" + "1: Yes" + "2: No", + examples=[2]) + + payment_method: Optional[str] = Field( + None, + description="Method of payment", + min_length=1, + max_length=48) +class Driver(BaseModel): + driver_id: Optional[str] = Field( + None, + description="Identifies the driver.", + min_length=1, + max_length=256, + pattern='^[a-zA-Z0-9_-]+$', + examples=["789"]) + profile_picture: Optional[HttpUrl] = Field( + None, + description="URL that contains the profile picture", + examples=["https://mfdz.de/driver/789/picture"]) + rating: Optional[int] = Field( + None, + description="Rating of the driver from 1 to 5." + "0 no rating yet", + ge=0, + le=5, + examples=[5]) + class Agency(BaseModel): id: str = Field( description="ID of the agency.", @@ -196,6 +281,17 @@ class Carpool(BaseModel): max_length=20, pattern='^[a-zA-Z0-9]+$', examples=["mfdz"]) + + driver: Optional[Driver] = Field( + None, + description="Driver data", + examples=[""" + { + "driver_id": "123", + "profile_picture": "https://mfdz.de/driver/789/picture", + "rating": 5 + } + """]) deeplink: HttpUrl = Field( description="Link to an information page providing detail information " @@ -246,7 +342,22 @@ class Carpool(BaseModel): "published.", examples=['A single date 2022-04-04 or a list of weekdays ["saturday", ' '"sunday"]']) - + route_color: Optional[str] = Field( + None, + pattern='^([0-9A-Fa-f]{6})$', + description="Route color designation that matches public facing material. " + "The color difference between route_color and route_text_color " + "should provide sufficient contrast when viewed on a black and " + "white screen.", + examples=["0039A6"]) + route_text_color: Optional[str] = Field( + None, + pattern='^([0-9A-Fa-f]{6})$', + description="Legible color to use for text drawn against a background of " + "route_color. The color difference between route_color and " + "route_text_color should provide sufficient contrast when " + "viewed on a black and white screen.", + examples=["D4D2D2"]) path: Optional[LineString] = Field( None, description="Optional route geometry as json LineString.") @@ -258,6 +369,18 @@ class Carpool(BaseModel): "purge outdated offers (e.g. older than 180 days). If not " "passed, the service may assume 'now'", examples=["2022-02-13T20:20:39+00:00"]) + additional_ridesharing_info: Optional[RidesharingInfo] = Field( + None, + description="Extension of GRFS to the GTFS standard", + examples=[""" + { + "number_free_seats": 2, + "creation_date": "2022-02-13T20:20:39+00:00", + "same_gender": 2, + "smoking": 1, + "luggage_size": 3 + } + """]) model_config = ConfigDict(json_schema_extra={ "title": "Carpool", # description ... @@ -268,18 +391,20 @@ class Carpool(BaseModel): "agency": "mfdz", "deeplink": "http://mfdz.de", "stops": [ - { - "id": "de:12073:900340137::2", "name": "ABC", - "lat": 45, "lon": 9 - }, - { - "id": "de:12073:900340137::3", "name": "XYZ", - "lat": 45, "lon": 9 - } + { + "name": "Stuttgart", + "lat": 48.783138, + "lon": 9.181288 + }, + { + "name": "Ulm", + "lat": 48.39892, + "lon": 9.98419 + } ], "departureTime": "12:34", - "departureDate": "2022-03-30", - "lastUpdated": "2022-03-30T12:34:00+00:00" - } + "departureDate": "2025-05-01", + "lastUpdated": "2025-02-18T12:34:00+00:00" + } """ }) diff --git a/amarillo/models/gtfs.py b/amarillo/models/gtfs.py deleted file mode 100644 index ee7f701..0000000 --- a/amarillo/models/gtfs.py +++ /dev/null @@ -1,29 +0,0 @@ -from collections import namedtuple -from datetime import timedelta - -GtfsFeedInfo = namedtuple('GtfsFeedInfo', 'feed_id feed_publisher_name feed_publisher_url feed_lang feed_version') -GtfsAgency = namedtuple('GtfsAgency', 'agency_id agency_name agency_url agency_timezone agency_lang agency_email') -GtfsRoute = namedtuple('GtfsRoute', 'agency_id route_id route_long_name route_type route_url route_short_name') -GtfsStop = namedtuple('GtfsStop', 'stop_id stop_lat stop_lon stop_name') -GtfsStopTime = namedtuple('GtfsStopTime', 'trip_id departure_time arrival_time stop_id stop_sequence pickup_type drop_off_type timepoint') -GtfsTrip = namedtuple('GtfsTrip', 'route_id trip_id service_id shape_id trip_headsign bikes_allowed') -GtfsCalendar = namedtuple('GtfsCalendar', 'service_id start_date end_date monday tuesday wednesday thursday friday saturday sunday') -GtfsCalendarDate = namedtuple('GtfsCalendarDate', 'service_id date exception_type') -GtfsShape = namedtuple('GtfsShape','shape_id shape_pt_lon shape_pt_lat shape_pt_sequence') - -# TODO Move to utils -class GtfsTimeDelta(timedelta): - def __str__(self): - seconds = self.total_seconds() - hours = seconds // 3600 - minutes = (seconds % 3600) // 60 - seconds = seconds % 60 - str = '{:02d}:{:02d}:{:02d}'.format(int(hours), int(minutes), int(seconds)) - return (str) - - def __add__(self, other): - if isinstance(other, timedelta): - return self.__class__(self.days + other.days, - self.seconds + other.seconds, - self.microseconds + other.microseconds) - return NotImplemented \ No newline at end of file diff --git a/amarillo/plugins/__init__.py b/amarillo/plugins/__init__.py new file mode 100644 index 0000000..69e3be5 --- /dev/null +++ b/amarillo/plugins/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/amarillo/routers/agency.py b/amarillo/routers/agency.py index cf50c47..67e98c6 100644 --- a/amarillo/routers/agency.py +++ b/amarillo/routers/agency.py @@ -2,12 +2,12 @@ import time from typing import List -from fastapi import APIRouter, HTTPException, status, Depends +from fastapi import APIRouter, HTTPException, status, Depends, BackgroundTasks from amarillo.models.Carpool import Carpool, Agency from amarillo.routers.agencyconf import verify_api_key, verify_admin_api_key, verify_permission_for_same_agency_or_admin # TODO should move this to service -from amarillo.routers.carpool import store_carpool, delete_agency_carpools_older_than +from amarillo.routers.carpool import store_carpool, delete_agency_carpools_older_than, enhance_trip from amarillo.services.agencies import AgencyService from amarillo.services.importing import Ride2GoImporter, NoiImporter from amarillo.utils.container import container @@ -52,6 +52,7 @@ async def get_agency(agency_id: str, admin_api_key: str = Depends(verify_api_key operation_id="sync", summary="Synchronizes all carpool offers", response_model=List[Carpool], + response_model_exclude_none=True, responses={ status.HTTP_200_OK: { "description": "Carpool created"}, @@ -60,7 +61,7 @@ async def get_agency(agency_id: str, admin_api_key: str = Depends(verify_api_key status.HTTP_500_INTERNAL_SERVER_ERROR: { "description": "Import error"} }) -async def sync(agency_id: str, requesting_agency_id: str = Depends(verify_api_key)) -> List[Carpool]: +async def sync(background_tasks: BackgroundTasks, agency_id: str, requesting_agency_id: str = Depends(verify_api_key)) -> List[Carpool]: await verify_permission_for_same_agency_or_admin(agency_id, requesting_agency_id) if agency_id == "ride2go": @@ -77,6 +78,10 @@ async def sync(agency_id: str, requesting_agency_id: str = Depends(verify_api_ke # Reduce current time by a minute to avoid inter process timestamp issues synced_files_older_than = time.time() - 60 result = [await store_carpool(cp) for cp in carpools] + + for cp in carpools: + background_tasks.add_task(enhance_trip, cp) + await delete_agency_carpools_older_than(agency_id, synced_files_older_than) return result except BaseException as e: diff --git a/amarillo/routers/agencyconf.py b/amarillo/routers/agencyconf.py index 036cd95..006162d 100644 --- a/amarillo/routers/agencyconf.py +++ b/amarillo/routers/agencyconf.py @@ -36,7 +36,9 @@ async def verify_admin_api_key(X_API_Key: str = Header(...)): async def verify_api_key(X_API_Key: str = Header(...)): agency_conf_service: AgencyConfService = container['agencyconf'] - return agency_conf_service.check_api_key(X_API_Key) + agency_id = agency_conf_service.check_api_key(X_API_Key) + logger.info(f"API key used: {agency_id}") + return agency_id # TODO Return code 403 Unauthoized (in response_status_codes as well...) async def verify_permission_for_same_agency_or_admin(agency_id_in_path_or_body, agency_id_from_api_key): diff --git a/amarillo/routers/carpool.py b/amarillo/routers/carpool.py index 1eeb96a..5548947 100644 --- a/amarillo/routers/carpool.py +++ b/amarillo/routers/carpool.py @@ -2,16 +2,22 @@ import json import os import os.path +from pathlib import Path import re from glob import glob -from fastapi import APIRouter, Body, Header, HTTPException, status, Depends +from fastapi import APIRouter, Body, Header, HTTPException, status, Depends, BackgroundTasks +import requests +from requests.exceptions import ConnectionError from datetime import datetime +from amarillo.utils.container import container from amarillo.models.Carpool import Carpool from amarillo.routers.agencyconf import verify_api_key, verify_permission_for_same_agency_or_admin from amarillo.tests.sampledata import examples - +from amarillo.services.hooks import run_on_create, run_on_delete +from amarillo.services.config import config +from amarillo.utils.utils import assert_folder_exists logger = logging.getLogger(__name__) @@ -20,26 +26,91 @@ tags=["carpool"] ) +#TODO: housekeeping for outdated trips + +def enhance_trip(carpool: Carpool): + try: + response = requests.post(f"{config.enhancer_url}", carpool.model_dump_json().encode('utf-8')) + response.raise_for_status() + enhanced_carpool = Carpool(**json.loads(response.content)) + + folder = f'data/enhanced/{carpool.agency}' + filename = f'{folder}/{carpool.id}.json' + + assert_folder_exists(folder) + with open(filename, 'w', encoding='utf-8') as f: + f.write(enhanced_carpool.model_dump_json()) + except ConnectionError: + handle_failed_carpool_enhancement(carpool) + logger.error("Could not connect to enhancer: make sure amarillo-enhancer is running and your ENHANCER_URL environment variable is configured correctly") + except requests.HTTPError as e: + handle_failed_carpool_enhancement(carpool) + logger.error(f"Error enhancing trip '{carpool.agency}:{carpool.id}': {e.response.status_code} {e.response.json()}") + except Exception as e: + handle_failed_carpool_enhancement(carpool) + logger.error(f"Error enhancing trip '{carpool.agency}:{carpool.id}': {e}") + +def handle_failed_carpool_enhancement(carpool: Carpool): + assert_folder_exists(f'data/failed/{carpool.agency}/') + with open(f'data/failed/{carpool.agency}/{carpool.id}.json', 'w', encoding='utf-8') as f: + f.write(carpool.model_dump_json()) + +def enhance_missing_carpools(): + logger.info(f"Enhancing missing restored trips...") + for agency_id in container['agencies'].agencies: + for carpool_file_name in glob(f'data/carpool/{agency_id}/*.json'): + carpool_id = Path(carpool_file_name).stem + carpool = _load_carpool_if_exists(agency_id, carpool_id, folder='data/carpool') + if not carpool: + # logger.warning(f"Failed loading carpool {agency_id}:{carpool_id}.") + continue + + existing_enhanced_carpool = _load_carpool_if_exists(agency_id, carpool.id, folder='data/enhanced') + if not existing_enhanced_carpool or existing_enhanced_carpool.lastUpdated != carpool.lastUpdated: + logger.info(f"Enhancing restored trip {carpool_file_name}") + try: + enhance_trip(carpool) + except Exception as e: + logger.warning("Issue during restore of carpool %s: %s", carpool_file_name, repr(e)) + +def carpool_exists(agency_id: str, carpool_id: str, folder: str ='data/enhanced'): + return os.path.exists(f"{folder}/{agency_id}/{carpool_id}.json") + +def _load_carpool_if_exists(agency_id: str, carpool_id: str, folder: str ='data/enhanced'): + if carpool_exists(agency_id, carpool_id, folder): + try: + return _load_carpool_from_path(f"{folder}/{agency_id}/{carpool_id}.json") + except Exception as e: + # An error on restore could be caused by model changes, + # in such a case, it need's to be recreated + logger.warning(f"Could not restore trip {folder}/{agency_id}/{carpool_id}.json: {repr(e)}") + + return None + + @router.post("/", operation_id="addcarpool", summary="Add a new or update existing carpool", description="Carpool object to be created or updated", response_model=Carpool, + response_model_exclude_none=True, responses={ status.HTTP_404_NOT_FOUND: { "description": "Agency does not exist"}, }) -async def post_carpool(carpool: Carpool = Body(..., examples=examples), +async def post_carpool(background_tasks: BackgroundTasks, carpool: Carpool = Body(..., examples=examples), requesting_agency_id: str = Depends(verify_api_key)) -> Carpool: await verify_permission_for_same_agency_or_admin(carpool.agency, requesting_agency_id) + background_tasks.add_task(run_on_create, carpool) + logger.info(f"POST trip {carpool.agency}:{carpool.id}.") await assert_agency_exists(carpool.agency) - await set_lastUpdated_if_unset(carpool) + await store_carpool(carpool) - await save_carpool(carpool) + background_tasks.add_task(enhance_trip, carpool) return carpool @@ -48,6 +119,7 @@ async def post_carpool(carpool: Carpool = Body(..., examples=examples), operation_id="getcarpoolById", summary="Find carpool by ID", response_model=Carpool, + response_model_exclude_none=True, description="Find carpool by ID", responses={ status.HTTP_404_NOT_FOUND: {"description": "Carpool not found"}, @@ -72,12 +144,14 @@ async def get_carpool(agency_id: str, carpool_id: str, api_key: str = Depends(ve "description": "Carpool or agency not found"}, }, ) -async def delete_carpool(agency_id: str, carpool_id: str, requesting_agency_id: str = Depends(verify_api_key)): +async def delete_carpool(background_tasks: BackgroundTasks, agency_id: str, carpool_id: str, requesting_agency_id: str = Depends(verify_api_key)): await verify_permission_for_same_agency_or_admin(agency_id, requesting_agency_id) logger.info(f"Delete trip {agency_id}:{carpool_id}.") await assert_agency_exists(agency_id) await assert_carpool_exists(agency_id, carpool_id) + cp = await load_carpool(agency_id, carpool_id) + background_tasks.add_task(run_on_delete, cp) return await _delete_carpool(agency_id, carpool_id) @@ -88,9 +162,16 @@ async def _delete_carpool(agency_id: str, carpool_id: str): # load and store, to receive pyinotify events and have file timestamp updated await save_carpool(cp, 'data/trash') logger.info(f"Saved carpool {agency_id}:{carpool_id} in trash.") - os.remove(f"data/carpool/{agency_id}/{carpool_id}.json") + try: + os.remove(f"data/carpool/{agency_id}/{carpool_id}.json") + os.remove(f"data/enhanced/{agency_id}/{carpool_id}.json", ) + except FileNotFoundError: + pass + async def store_carpool(carpool: Carpool) -> Carpool: + carpool_exists = os.path.exists(f"data/carpool/{carpool.agency}/{carpool.id}.json") + await set_lastUpdated_if_unset(carpool) await save_carpool(carpool) @@ -101,20 +182,22 @@ async def set_lastUpdated_if_unset(carpool): carpool.lastUpdated = datetime.now() -async def load_carpool(agency_id, carpool_id) -> Carpool: - with open(f'data/carpool/{agency_id}/{carpool_id}.json', 'r', encoding='utf-8') as f: +async def load_carpool(agency_id, carpool_id, folder: str ='data/carpool') -> Carpool: + return _load_carpool_from_path(f'{folder}/{agency_id}/{carpool_id}.json') + +def _load_carpool_from_path(path: str): + with open(path, 'r', encoding='utf-8') as f: dict = json.load(f) carpool = Carpool(**dict) return carpool - async def save_carpool(carpool, folder: str = 'data/carpool'): with open(f'{folder}/{carpool.agency}/{carpool.id}.json', 'w', encoding='utf-8') as f: f.write(carpool.json()) async def assert_agency_exists(agency_id: str): - agency_exists = os.path.exists(f"conf/agency/{agency_id}.json") + agency_exists = os.path.exists(f"data/agency/{agency_id}.json") if not agency_exists: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, @@ -134,4 +217,6 @@ async def delete_agency_carpools_older_than(agency_id, timestamp): if os.path.getmtime(carpool_file_name) < timestamp: m = re.search(r'([a-zA-Z0-9_-]+)\.json$', carpool_file_name) # TODO log deletion + cp = await load_carpool(agency_id, m[1]) + run_on_delete(cp) await _delete_carpool(agency_id, m[1]) diff --git a/amarillo/routers/region.py b/amarillo/routers/region.py index 50da511..a530522 100644 --- a/amarillo/routers/region.py +++ b/amarillo/routers/region.py @@ -45,7 +45,7 @@ async def get_region(region_id: str) -> Region: return region def _assert_region_exists(region_id: str) -> Region: - regions: regionService = container['regions'] + regions: RegionService = container['regions'] region = regions.get_region(region_id) region_exists = region is not None @@ -55,34 +55,3 @@ def _assert_region_exists(region_id: str) -> Region: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=message) return region - -@router.get("/{region_id}/gtfs", - summary="Return GTFS Feed for this region", - response_description="GTFS-Feed (zip-file)", - response_class=FileResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"description": "Region not found"}, - } - ) -async def get_file(region_id: str, user: str = Depends(verify_admin_api_key)): - _assert_region_exists(region_id) - return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfs.zip') - -@router.get("/{region_id}/gtfs-rt", - summary="Return GTFS-RT Feed for this region", - response_description="GTFS-RT-Feed", - response_class=FileResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"description": "Region not found"}, - status.HTTP_400_BAD_REQUEST: {"description": "Bad request, e.g. because format is not supported, i.e. neither protobuf nor json."} - } - ) -async def get_file(region_id: str, format: str = 'protobuf', user: str = Depends(verify_admin_api_key)): - _assert_region_exists(region_id) - if format == 'json': - return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfsrt.json') - elif format == 'protobuf': - return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfsrt.pbf') - else: - message = "Specified format is not supported, i.e. neither protobuf nor json." - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) diff --git a/amarillo/services/agencies.py b/amarillo/services/agencies.py index 47baa5a..e7450aa 100644 --- a/amarillo/services/agencies.py +++ b/amarillo/services/agencies.py @@ -12,8 +12,7 @@ class AgencyService: def __init__(self): self.agencies: Dict[str, Agency] = {} - - for agency_file_name in glob('conf/agency/*.json'): + for agency_file_name in glob('data/agency/*.json'): with open(agency_file_name) as agency_file: dict = json.load(agency_file) agency = Agency(**dict) diff --git a/amarillo/services/config.py b/amarillo/services/config.py index 1dd620a..867c169 100644 --- a/amarillo/services/config.py +++ b/amarillo/services/config.py @@ -1,4 +1,5 @@ from typing import List +from pydantic import ConfigDict from pydantic_settings import BaseSettings @@ -8,6 +9,9 @@ class Config(BaseSettings): ride2go_query_data: str env: str = 'DEV' graphhopper_base_url: str = 'https://api.mfdz.de/gh' - stop_sources_file: str = 'conf/stop_sources.json' + stop_sources_file: str = 'data/stop_sources.json' + enhancer_url: str = 'http://localhost:8001' + + model_config = ConfigDict(extra='allow') # Allow plugins to add extra values config = Config(_env_file='config', _env_file_encoding='utf-8') diff --git a/amarillo/services/gtfs.py b/amarillo/services/gtfs.py deleted file mode 100644 index c45a626..0000000 --- a/amarillo/services/gtfs.py +++ /dev/null @@ -1,137 +0,0 @@ -import amarillo.services.gtfsrt.gtfs_realtime_pb2 as gtfs_realtime_pb2 -import amarillo.services.gtfsrt.realtime_extension_pb2 as mfdzrte -from amarillo.services.gtfs_constants import * -from google.protobuf.json_format import MessageToDict -from google.protobuf.json_format import ParseDict -from datetime import datetime, timedelta -import json -import re -import time - -class GtfsRtProducer(): - - def __init__(self, trip_store): - self.trip_store = trip_store - - def generate_feed(self, time, format='protobuf', bbox=None): - # See https://developers.google.com/transit/gtfs-realtime/reference - # https://github.com/mfdz/carpool-gtfs-rt/blob/master/src/main/java/de/mfdz/resource/CarpoolResource.java - gtfsrt_dict = { - 'header': { - 'gtfsRealtimeVersion': '1.0', - 'timestamp': int(time) - }, - 'entity': self._get_trip_updates(bbox) - } - feed = gtfs_realtime_pb2.FeedMessage() - ParseDict(gtfsrt_dict, feed) - - if "message" == format.lower(): - return feed - elif "json" == format.lower(): - return MessageToDict(feed) - else: - return feed.SerializeToString() - - def export_feed(self, timestamp, file_path, bbox=None): - """ - Exports gtfs-rt feed as .json and .pbf file to file_path - """ - feed = self.generate_feed(timestamp, "message", bbox) - with open(f"{file_path}.pbf", "wb") as f: - f.write(feed.SerializeToString()) - with open(f"{file_path}.json", "w") as f: - json.dump(MessageToDict(feed), f) - - def _get_trip_updates(self, bbox = None): - trips = [] - trips.extend(self._get_added(bbox)) - trips.extend(self._get_deleted(bbox)) - trip_updates = [] - for num, trip in enumerate(trips): - trip_updates.append( { - 'id': f'carpool-update-{num}', - 'tripUpdate': trip - } - ) - return trip_updates - - def _get_deleted(self, bbox = None): - return self._get_updates( - self.trip_store.recently_deleted_trips(), - self._as_delete_updates, - bbox) - - def _get_added(self, bbox = None): - return self._get_updates( - self.trip_store.recently_added_trips(), - self._as_added_updates, - bbox) - - def _get_updates(self, trips, update_func, bbox = None): - updates = [] - today = datetime.today() - for t in trips: - if bbox == None or t.intersects(bbox): - updates.extend(update_func(t, today)) - return updates - - def _as_delete_updates(self, trip, fromdate): - return [{ - 'trip': { - 'tripId': trip.trip_id, - 'startTime': trip.start_time_str(), - 'startDate': trip_date, - 'scheduleRelationship': 'CANCELED', - 'routeId': trip.trip_id - } - } for trip_date in trip.next_trip_dates(fromdate)] - - def _to_seconds(self, fromdate, stop_time): - startdate = datetime.strptime(fromdate, '%Y%m%d') - m = re.search(r'(\d+):(\d+):(\d+)', stop_time) - delta = timedelta( - hours=int(m.group(1)), - minutes=int(m.group(2)), - seconds=int(m.group(3))) - return time.mktime((startdate + delta).timetuple()) - - def _to_stop_times(self, trip, fromdate): - return [{ - 'stopSequence': stoptime.stop_sequence, - 'arrival': { - 'time': self._to_seconds(fromdate, stoptime.arrival_time), - 'uncertainty': MFDZ_DEFAULT_UNCERTAINITY - }, - 'departure': { - 'time': self._to_seconds(fromdate, stoptime.departure_time), - 'uncertainty': MFDZ_DEFAULT_UNCERTAINITY - }, - 'stopId': stoptime.stop_id, - 'scheduleRelationship': 'SCHEDULED', - 'stop_time_properties': { - '[transit_realtime.stop_time_properties]': { - 'dropoffType': 'COORDINATE_WITH_DRIVER' if stoptime.drop_off_type == STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER else 'NONE', - 'pickupType': 'COORDINATE_WITH_DRIVER' if stoptime.pickup_type == STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER else 'NONE' - } - } - } - for stoptime in trip.stop_times] - - def _as_added_updates(self, trip, fromdate): - return [{ - 'trip': { - 'tripId': trip.trip_id, - 'startTime': trip.start_time_str(), - 'startDate': trip_date, - 'scheduleRelationship': 'ADDED', - 'routeId': trip.trip_id, - '[transit_realtime.trip_descriptor]': { - 'routeUrl' : trip.url, - 'agencyId' : trip.agency, - 'route_long_name' : trip.route_long_name(), - 'route_type': RIDESHARING_ROUTE_TYPE - } - }, - 'stopTimeUpdate': self._to_stop_times(trip, trip_date) - } for trip_date in trip.next_trip_dates(fromdate)] diff --git a/amarillo/services/gtfs_constants.py b/amarillo/services/gtfs_constants.py deleted file mode 100644 index 1e8f3af..0000000 --- a/amarillo/services/gtfs_constants.py +++ /dev/null @@ -1,14 +0,0 @@ -# Constants - -NO_BIKES_ALLOWED = 2 -RIDESHARING_ROUTE_TYPE = 1551 -CALENDAR_DATES_EXCEPTION_TYPE_ADDED = 1 -CALENDAR_DATES_EXCEPTION_TYPE_REMOVED = 2 -STOP_TIMES_STOP_TYPE_REGULARLY = 0 -STOP_TIMES_STOP_TYPE_NONE = 1 -STOP_TIMES_STOP_TYPE_PHONE_AGENCY = 2 -STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER = 3 -STOP_TIMES_TIMEPOINT_APPROXIMATE = 0 -STOP_TIMES_TIMEPOINT_EXACT = 1 - -MFDZ_DEFAULT_UNCERTAINITY = 600 \ No newline at end of file diff --git a/amarillo/services/gtfs_export.py b/amarillo/services/gtfs_export.py deleted file mode 100644 index fb39425..0000000 --- a/amarillo/services/gtfs_export.py +++ /dev/null @@ -1,229 +0,0 @@ - -from collections.abc import Iterable -from datetime import datetime, timedelta -from zipfile import ZipFile -import csv -import gettext -import logging -import re - -from amarillo.utils.utils import assert_folder_exists -from amarillo.models.gtfs import GtfsTimeDelta, GtfsFeedInfo, GtfsAgency, GtfsRoute, GtfsStop, GtfsStopTime, GtfsTrip, GtfsCalendar, GtfsCalendarDate, GtfsShape -from amarillo.services.stops import is_carpooling_stop -from amarillo.services.gtfs_constants import * - - -logger = logging.getLogger(__name__) - -class GtfsExport: - - stops_counter = 0 - trips_counter = 0 - routes_counter = 0 - - stored_stops = {} - - def __init__(self, agencies, feed_info, ridestore, stopstore, bbox = None): - self.stops = {} - self.routes = [] - self.calendar_dates = [] - self.calendar = [] - self.trips = [] - self.stop_times = [] - self.calendar = [] - self.shapes = [] - self.agencies = agencies - self.feed_info = feed_info - self.localized_to = " nach " - self.localized_short_name = "Mitfahrgelegenheit" - self.stopstore = stopstore - self.ridestore = ridestore - self.bbox = bbox - - def export(self, gtfszip_filename, gtfsfolder): - assert_folder_exists(gtfsfolder) - self._prepare_gtfs_feed(self.ridestore, self.stopstore) - self._write_csvfile(gtfsfolder, 'agency.txt', self.agencies) - self._write_csvfile(gtfsfolder, 'feed_info.txt', self.feed_info) - self._write_csvfile(gtfsfolder, 'routes.txt', self.routes) - self._write_csvfile(gtfsfolder, 'trips.txt', self.trips) - self._write_csvfile(gtfsfolder, 'calendar.txt', self.calendar) - self._write_csvfile(gtfsfolder, 'calendar_dates.txt', self.calendar_dates) - self._write_csvfile(gtfsfolder, 'stops.txt', self.stops.values()) - self._write_csvfile(gtfsfolder, 'stop_times.txt', self.stop_times) - self._write_csvfile(gtfsfolder, 'shapes.txt', self.shapes) - self._zip_files(gtfszip_filename, gtfsfolder) - - def _zip_files(self, gtfszip_filename, gtfsfolder): - gtfsfiles = ['agency.txt', 'feed_info.txt', 'routes.txt', 'trips.txt', - 'calendar.txt', 'calendar_dates.txt', 'stops.txt', 'stop_times.txt', 'shapes.txt'] - with ZipFile(gtfszip_filename, 'w') as gtfszip: - for gtfsfile in gtfsfiles: - gtfszip.write(gtfsfolder+'/'+gtfsfile, gtfsfile) - - def _prepare_gtfs_feed(self, ridestore, stopstore): - """ - Prepares all gtfs objects in memory before they are written - to their respective streams. - - For all wellknown stops a GTFS stop is created and - afterwards all ride offers are transformed into their - gtfs equivalents. - """ - for stopSet in stopstore.stopsDataFrames: - for stop in stopSet["stops"].itertuples(): - self._load_stored_stop(stop) - cloned_trips = dict(ridestore.trips) - for url, trip in cloned_trips.items(): - if self.bbox is None or trip.intersects(self.bbox): - self._convert_trip(trip) - - def _convert_trip(self, trip): - self.routes_counter += 1 - self.routes.append(self._create_route(trip)) - self.calendar.append(self._create_calendar(trip)) - if not trip.runs_regularly: - self.calendar_dates.append(self._create_calendar_date(trip)) - self.trips.append(self._create_trip(trip, self.routes_counter)) - self._append_stops_and_stop_times(trip) - self._append_shapes(trip, self.routes_counter) - - def _trip_headsign(self, destination): - destination = destination.replace('(Deutschland)', '') - destination = destination.replace(', Deutschland', '') - appendix = '' - if 'Schweiz' in destination or 'Switzerland' in destination: - appendix = ', Schweiz' - destination = destination.replace('(Schweiz)', '') - destination = destination.replace(', Schweiz', '') - destination = destination.replace('(Switzerland)', '') - - try: - matches = re.match(r"(.*,)? ?(\d{4,5})? ?(.*)", destination) - - match = matches.group(3).strip() if matches != None else destination.strip() - if match[-1]==')' and not '(' in match: - match = match[0:-1] - - return match + appendix - except Exception as ex: - logger.error("error for "+destination ) - logger.exception(ex) - return destination - - def _create_route(self, trip): - return GtfsRoute(trip.agency, trip.trip_id, trip.route_long_name(), RIDESHARING_ROUTE_TYPE, trip.url, "") - - def _create_calendar(self, trip): - # TODO currently, calendar is not provided by Fahrgemeinschaft.de interface. - # We could apply some heuristics like requesting multiple days and extrapolate - # if multiple trips are found, but better would be to have these provided by the - # offical interface. Then validity periods should be provided as well (not - # sure if these are available) - # For fahrgemeinschaft.de, regurlar trips are recognizable via their url - # which contains "regelmaessig". However, we don't know on which days of the week, - # nor until when. As a first guess, if datetime is a mo-fr, we assume each workday, - # if it's sa/su, only this... - - feed_start_date = datetime.today() - stop_date = self._convert_stop_date(feed_start_date) - return GtfsCalendar(trip.trip_id, stop_date, self._convert_stop_date(feed_start_date + timedelta(days=31)), *(trip.weekdays)) - - def _create_calendar_date(self, trip): - return GtfsCalendarDate(trip.trip_id, self._convert_stop_date(trip.start), CALENDAR_DATES_EXCEPTION_TYPE_ADDED) - - def _create_trip(self, trip, shape_id): - return GtfsTrip(trip.trip_id, trip.trip_id, trip.trip_id, shape_id, trip.trip_headsign, NO_BIKES_ALLOWED) - - def _convert_stop(self, stop): - """ - Converts a stop represented as pandas row to a gtfs stop. - Expected attributes of stop: id, stop_name, x, y (in wgs84) - """ - if stop.id: - id = stop.id - else: - self.stops_counter += 1 - id = "tmp-{}".format(self.stops_counter) - - stop_name = "k.A." if stop.stop_name is None else stop.stop_name - return GtfsStop(id, stop.y, stop.x, stop_name) - - def _append_stops_and_stop_times(self, trip): - # Assumptions: - # arrival_time = departure_time - # pickup_type, drop_off_type for origin: = coordinate/none - # pickup_type, drop_off_type for destination: = none/coordinate - # timepoint = approximate for origin and destination (not sure what consequences this might have for trip planners) - for stop_time in trip.stop_times: - # retrieve stop from stored_stops and mark it to be exported - wkn_stop = self.stored_stops.get(stop_time.stop_id) - if not wkn_stop: - logger.warning("No stop found in stop_store for %s. Will skip stop_time %s of trip %s", stop_time.stop_id, stop_time.stop_sequence, trip.trip_id) - else: - self.stops[stop_time.stop_id] = wkn_stop - # Append stop_time - self.stop_times.append(stop_time) - - def _append_shapes(self, trip, shape_id): - counter = 0 - for point in trip.path.coordinates: - counter += 1 - self.shapes.append(GtfsShape(shape_id, point[0], point[1], counter)) - - def _stop_hash(self, stop): - return "{}#{}#{}".format(stop.stop_name,stop.x,stop.y) - - def _should_always_export(self, stop): - """ - Returns true, if the given stop shall be exported to GTFS, - regardless, if it's part of a trip or not. - - This is necessary, as potential stops are required - to be part of the GTFS to be referenced later on - by dynamicly added trips. - """ - if self.bbox: - return (self.bbox[0] <= stop.stop_lon <= self.bbox[2] and - self.bbox[1] <= stop.stop_lat <= self.bbox[3]) - else: - return is_carpooling_stop(stop.stop_id, stop.stop_name) - - def _load_stored_stop(self, stop): - gtfsstop = self._convert_stop(stop) - stop_hash = self._stop_hash(stop) - self.stored_stops[gtfsstop.stop_id] = gtfsstop - if self._should_always_export(gtfsstop): - self.stops[gtfsstop.stop_id] = gtfsstop - - def _get_stop_by_hash(self, stop_hash): - return self.stops.get(stop_hash, self.stored_stops.get(stop_hash)) - - def _get_or_create_stop(self, stop): - stop_hash = self._stop_hash(stop) - gtfsstop = self.stops.get(stop_hash) - if gtfsstop is None: - gtfsstop = self.stored_stops.get(stop_hash, self._convert_stop(stop)) - self.stops[stop_hash] = gtfsstop - return gtfsstop - - def _convert_stop_date(self, date_time): - return date_time.strftime("%Y%m%d") - - def _write_csvfile(self, gtfsfolder, filename, content): - with open(gtfsfolder+"/"+filename, 'w', newline="\n", encoding="utf-8") as csvfile: - self._write_csv(csvfile, content) - - def _write_csv(self, csvfile, content): - if hasattr(content, '_fields'): - writer = csv.DictWriter(csvfile, content._fields) - writer.writeheader() - writer.writerow(content._asdict()) - else: - if content: - writer = csv.DictWriter(csvfile, next(iter(content))._fields) - writer.writeheader() - for record in content: - writer.writerow(record._asdict()) - - \ No newline at end of file diff --git a/amarillo/services/gtfs_generator.py b/amarillo/services/gtfs_generator.py deleted file mode 100644 index 113d5db..0000000 --- a/amarillo/services/gtfs_generator.py +++ /dev/null @@ -1,71 +0,0 @@ -from amarillo.models.Carpool import Region -from amarillo.services.gtfs_export import GtfsExport, GtfsFeedInfo, GtfsAgency -from amarillo.services.gtfs import GtfsRtProducer -from amarillo.utils.container import container -from glob import glob -import json -import schedule -import threading -import time -import logging -from datetime import date, timedelta - -logger = logging.getLogger(__name__) - -regions = {} -for region_file_name in glob('conf/region/*.json'): - with open(region_file_name) as region_file: - dict = json.load(region_file) - region = Region(**dict) - region_id = region.id - regions[region_id] = region - -agencies = [] -for agency_file_name in glob('conf/agency/*.json'): - with open(agency_file_name) as agency_file: - dict = json.load(agency_file) - agency = GtfsAgency(dict["id"], dict["name"], dict["url"], dict["timezone"], dict["lang"], dict["email"]) - agency_id = agency.agency_id - agencies.append(agency) - -def run_schedule(): - while 1: - try: - schedule.run_pending() - except Exception as e: - logger.exception(e) - time.sleep(1) - -def midnight(): - container['stops_store'].load_stop_sources() - container['trips_store'].unflag_unrecent_updates() - container['carpools'].purge_outdated_offers() - generate_gtfs() - -def generate_gtfs(): - logger.info("Generate GTFS") - - for region in regions.values(): - # TODO make feed producer infos configurable - feed_info = GtfsFeedInfo('mfdz', 'MITFAHR|DE|ZENTRALE', 'http://www.mitfahrdezentrale.de', 'de', 1) - exporter = GtfsExport( - agencies, - feed_info, - container['trips_store'], - container['stops_store'], - region.bbox) - exporter.export(f"data/gtfs/amarillo.{region.id}.gtfs.zip", "data/tmp/") - -def generate_gtfs_rt(): - logger.info("Generate GTFS-RT") - producer = GtfsRtProducer(container['trips_store']) - for region in regions.values(): - rt = producer.export_feed(time.time(), f"data/gtfs/amarillo.{region.id}.gtfsrt", bbox=region.bbox) - -def start_schedule(): - schedule.every().day.at("00:00").do(midnight) - schedule.every(60).seconds.do(generate_gtfs_rt) - # Create all feeds once at startup - schedule.run_all() - job_thread = threading.Thread(target=run_schedule, daemon=True) - job_thread.start() \ No newline at end of file diff --git a/amarillo/services/gtfsrt/__init__.py b/amarillo/services/gtfsrt/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/amarillo/services/gtfsrt/gtfs_realtime_pb2.py b/amarillo/services/gtfsrt/gtfs_realtime_pb2.py deleted file mode 100644 index 4e10463..0000000 --- a/amarillo/services/gtfsrt/gtfs_realtime_pb2.py +++ /dev/null @@ -1,80 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: gtfs-realtime.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13gtfs-realtime.proto\x12\x10transit_realtime\"y\n\x0b\x46\x65\x65\x64Message\x12,\n\x06header\x18\x01 \x02(\x0b\x32\x1c.transit_realtime.FeedHeader\x12,\n\x06\x65ntity\x18\x02 \x03(\x0b\x32\x1c.transit_realtime.FeedEntity*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xd7\x01\n\nFeedHeader\x12\x1d\n\x15gtfs_realtime_version\x18\x01 \x02(\t\x12Q\n\x0eincrementality\x18\x02 \x01(\x0e\x32+.transit_realtime.FeedHeader.Incrementality:\x0c\x46ULL_DATASET\x12\x11\n\ttimestamp\x18\x03 \x01(\x04\"4\n\x0eIncrementality\x12\x10\n\x0c\x46ULL_DATASET\x10\x00\x12\x10\n\x0c\x44IFFERENTIAL\x10\x01*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xd2\x01\n\nFeedEntity\x12\n\n\x02id\x18\x01 \x02(\t\x12\x19\n\nis_deleted\x18\x02 \x01(\x08:\x05\x66\x61lse\x12\x31\n\x0btrip_update\x18\x03 \x01(\x0b\x32\x1c.transit_realtime.TripUpdate\x12\x32\n\x07vehicle\x18\x04 \x01(\x0b\x32!.transit_realtime.VehiclePosition\x12&\n\x05\x61lert\x18\x05 \x01(\x0b\x32\x17.transit_realtime.Alert*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\x82\x08\n\nTripUpdate\x12.\n\x04trip\x18\x01 \x02(\x0b\x32 .transit_realtime.TripDescriptor\x12\x34\n\x07vehicle\x18\x03 \x01(\x0b\x32#.transit_realtime.VehicleDescriptor\x12\x45\n\x10stop_time_update\x18\x02 \x03(\x0b\x32+.transit_realtime.TripUpdate.StopTimeUpdate\x12\x11\n\ttimestamp\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65lay\x18\x05 \x01(\x05\x12\x44\n\x0ftrip_properties\x18\x06 \x01(\x0b\x32+.transit_realtime.TripUpdate.TripProperties\x1aQ\n\rStopTimeEvent\x12\r\n\x05\x64\x65lay\x18\x01 \x01(\x05\x12\x0c\n\x04time\x18\x02 \x01(\x03\x12\x13\n\x0buncertainty\x18\x03 \x01(\x05*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\x1a\xa0\x04\n\x0eStopTimeUpdate\x12\x15\n\rstop_sequence\x18\x01 \x01(\r\x12\x0f\n\x07stop_id\x18\x04 \x01(\t\x12;\n\x07\x61rrival\x18\x02 \x01(\x0b\x32*.transit_realtime.TripUpdate.StopTimeEvent\x12=\n\tdeparture\x18\x03 \x01(\x0b\x32*.transit_realtime.TripUpdate.StopTimeEvent\x12j\n\x15schedule_relationship\x18\x05 \x01(\x0e\x32@.transit_realtime.TripUpdate.StopTimeUpdate.ScheduleRelationship:\tSCHEDULED\x12\\\n\x14stop_time_properties\x18\x06 \x01(\x0b\x32>.transit_realtime.TripUpdate.StopTimeUpdate.StopTimeProperties\x1a>\n\x12StopTimeProperties\x12\x18\n\x10\x61ssigned_stop_id\x18\x01 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"P\n\x14ScheduleRelationship\x12\r\n\tSCHEDULED\x10\x00\x12\x0b\n\x07SKIPPED\x10\x01\x12\x0b\n\x07NO_DATA\x10\x02\x12\x0f\n\x0bUNSCHEDULED\x10\x03*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\x1aY\n\x0eTripProperties\x12\x0f\n\x07trip_id\x18\x01 \x01(\t\x12\x12\n\nstart_date\x18\x02 \x01(\t\x12\x12\n\nstart_time\x18\x03 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xdf\t\n\x0fVehiclePosition\x12.\n\x04trip\x18\x01 \x01(\x0b\x32 .transit_realtime.TripDescriptor\x12\x34\n\x07vehicle\x18\x08 \x01(\x0b\x32#.transit_realtime.VehicleDescriptor\x12,\n\x08position\x18\x02 \x01(\x0b\x32\x1a.transit_realtime.Position\x12\x1d\n\x15\x63urrent_stop_sequence\x18\x03 \x01(\r\x12\x0f\n\x07stop_id\x18\x07 \x01(\t\x12Z\n\x0e\x63urrent_status\x18\x04 \x01(\x0e\x32\x33.transit_realtime.VehiclePosition.VehicleStopStatus:\rIN_TRANSIT_TO\x12\x11\n\ttimestamp\x18\x05 \x01(\x04\x12K\n\x10\x63ongestion_level\x18\x06 \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.CongestionLevel\x12K\n\x10occupancy_status\x18\t \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.OccupancyStatus\x12\x1c\n\x14occupancy_percentage\x18\n \x01(\r\x12Q\n\x16multi_carriage_details\x18\x0b \x03(\x0b\x32\x31.transit_realtime.VehiclePosition.CarriageDetails\x1a\xd9\x01\n\x0f\x43\x61rriageDetails\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05label\x18\x02 \x01(\t\x12^\n\x10occupancy_status\x18\x03 \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.OccupancyStatus:\x11NO_DATA_AVAILABLE\x12 \n\x14occupancy_percentage\x18\x04 \x01(\x05:\x02-1\x12\x19\n\x11\x63\x61rriage_sequence\x18\x05 \x01(\r*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"G\n\x11VehicleStopStatus\x12\x0f\n\x0bINCOMING_AT\x10\x00\x12\x0e\n\nSTOPPED_AT\x10\x01\x12\x11\n\rIN_TRANSIT_TO\x10\x02\"}\n\x0f\x43ongestionLevel\x12\x1c\n\x18UNKNOWN_CONGESTION_LEVEL\x10\x00\x12\x14\n\x10RUNNING_SMOOTHLY\x10\x01\x12\x0f\n\x0bSTOP_AND_GO\x10\x02\x12\x0e\n\nCONGESTION\x10\x03\x12\x15\n\x11SEVERE_CONGESTION\x10\x04\"\xd9\x01\n\x0fOccupancyStatus\x12\t\n\x05\x45MPTY\x10\x00\x12\x18\n\x14MANY_SEATS_AVAILABLE\x10\x01\x12\x17\n\x13\x46\x45W_SEATS_AVAILABLE\x10\x02\x12\x16\n\x12STANDING_ROOM_ONLY\x10\x03\x12\x1e\n\x1a\x43RUSHED_STANDING_ROOM_ONLY\x10\x04\x12\x08\n\x04\x46ULL\x10\x05\x12\x1c\n\x18NOT_ACCEPTING_PASSENGERS\x10\x06\x12\x15\n\x11NO_DATA_AVAILABLE\x10\x07\x12\x11\n\rNOT_BOARDABLE\x10\x08*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\x80\t\n\x05\x41lert\x12\x32\n\ractive_period\x18\x01 \x03(\x0b\x32\x1b.transit_realtime.TimeRange\x12\x39\n\x0finformed_entity\x18\x05 \x03(\x0b\x32 .transit_realtime.EntitySelector\x12;\n\x05\x63\x61use\x18\x06 \x01(\x0e\x32\x1d.transit_realtime.Alert.Cause:\rUNKNOWN_CAUSE\x12>\n\x06\x65\x66\x66\x65\x63t\x18\x07 \x01(\x0e\x32\x1e.transit_realtime.Alert.Effect:\x0eUNKNOWN_EFFECT\x12/\n\x03url\x18\x08 \x01(\x0b\x32\".transit_realtime.TranslatedString\x12\x37\n\x0bheader_text\x18\n \x01(\x0b\x32\".transit_realtime.TranslatedString\x12<\n\x10\x64\x65scription_text\x18\x0b \x01(\x0b\x32\".transit_realtime.TranslatedString\x12;\n\x0ftts_header_text\x18\x0c \x01(\x0b\x32\".transit_realtime.TranslatedString\x12@\n\x14tts_description_text\x18\r \x01(\x0b\x32\".transit_realtime.TranslatedString\x12O\n\x0eseverity_level\x18\x0e \x01(\x0e\x32%.transit_realtime.Alert.SeverityLevel:\x10UNKNOWN_SEVERITY\"\xd8\x01\n\x05\x43\x61use\x12\x11\n\rUNKNOWN_CAUSE\x10\x01\x12\x0f\n\x0bOTHER_CAUSE\x10\x02\x12\x15\n\x11TECHNICAL_PROBLEM\x10\x03\x12\n\n\x06STRIKE\x10\x04\x12\x11\n\rDEMONSTRATION\x10\x05\x12\x0c\n\x08\x41\x43\x43IDENT\x10\x06\x12\x0b\n\x07HOLIDAY\x10\x07\x12\x0b\n\x07WEATHER\x10\x08\x12\x0f\n\x0bMAINTENANCE\x10\t\x12\x10\n\x0c\x43ONSTRUCTION\x10\n\x12\x13\n\x0fPOLICE_ACTIVITY\x10\x0b\x12\x15\n\x11MEDICAL_EMERGENCY\x10\x0c\"\xdd\x01\n\x06\x45\x66\x66\x65\x63t\x12\x0e\n\nNO_SERVICE\x10\x01\x12\x13\n\x0fREDUCED_SERVICE\x10\x02\x12\x16\n\x12SIGNIFICANT_DELAYS\x10\x03\x12\n\n\x06\x44\x45TOUR\x10\x04\x12\x16\n\x12\x41\x44\x44ITIONAL_SERVICE\x10\x05\x12\x14\n\x10MODIFIED_SERVICE\x10\x06\x12\x10\n\x0cOTHER_EFFECT\x10\x07\x12\x12\n\x0eUNKNOWN_EFFECT\x10\x08\x12\x0e\n\nSTOP_MOVED\x10\t\x12\r\n\tNO_EFFECT\x10\n\x12\x17\n\x13\x41\x43\x43\x45SSIBILITY_ISSUE\x10\x0b\"H\n\rSeverityLevel\x12\x14\n\x10UNKNOWN_SEVERITY\x10\x01\x12\x08\n\x04INFO\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06SEVERE\x10\x04*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"7\n\tTimeRange\x12\r\n\x05start\x18\x01 \x01(\x04\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x04*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"q\n\x08Position\x12\x10\n\x08latitude\x18\x01 \x02(\x02\x12\x11\n\tlongitude\x18\x02 \x02(\x02\x12\x0f\n\x07\x62\x65\x61ring\x18\x03 \x01(\x02\x12\x10\n\x08odometer\x18\x04 \x01(\x01\x12\r\n\x05speed\x18\x05 \x01(\x02*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xcd\x02\n\x0eTripDescriptor\x12\x0f\n\x07trip_id\x18\x01 \x01(\t\x12\x10\n\x08route_id\x18\x05 \x01(\t\x12\x14\n\x0c\x64irection_id\x18\x06 \x01(\r\x12\x12\n\nstart_time\x18\x02 \x01(\t\x12\x12\n\nstart_date\x18\x03 \x01(\t\x12T\n\x15schedule_relationship\x18\x04 \x01(\x0e\x32\x35.transit_realtime.TripDescriptor.ScheduleRelationship\"t\n\x14ScheduleRelationship\x12\r\n\tSCHEDULED\x10\x00\x12\t\n\x05\x41\x44\x44\x45\x44\x10\x01\x12\x0f\n\x0bUNSCHEDULED\x10\x02\x12\x0c\n\x08\x43\x41NCELED\x10\x03\x12\x13\n\x0bREPLACEMENT\x10\x05\x1a\x02\x08\x01\x12\x0e\n\nDUPLICATED\x10\x06*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"U\n\x11VehicleDescriptor\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05label\x18\x02 \x01(\t\x12\x15\n\rlicense_plate\x18\x03 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xb0\x01\n\x0e\x45ntitySelector\x12\x11\n\tagency_id\x18\x01 \x01(\t\x12\x10\n\x08route_id\x18\x02 \x01(\t\x12\x12\n\nroute_type\x18\x03 \x01(\x05\x12.\n\x04trip\x18\x04 \x01(\x0b\x32 .transit_realtime.TripDescriptor\x12\x0f\n\x07stop_id\x18\x05 \x01(\t\x12\x14\n\x0c\x64irection_id\x18\x06 \x01(\r*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xa6\x01\n\x10TranslatedString\x12\x43\n\x0btranslation\x18\x01 \x03(\x0b\x32..transit_realtime.TranslatedString.Translation\x1a=\n\x0bTranslation\x12\x0c\n\x04text\x18\x01 \x02(\t\x12\x10\n\x08language\x18\x02 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90NB\x1d\n\x1b\x63om.google.transit.realtime') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'gtfs_realtime_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\033com.google.transit.realtime' - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP.values_by_name["REPLACEMENT"]._options = None - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP.values_by_name["REPLACEMENT"]._serialized_options = b'\010\001' - _FEEDMESSAGE._serialized_start=41 - _FEEDMESSAGE._serialized_end=162 - _FEEDHEADER._serialized_start=165 - _FEEDHEADER._serialized_end=380 - _FEEDHEADER_INCREMENTALITY._serialized_start=312 - _FEEDHEADER_INCREMENTALITY._serialized_end=364 - _FEEDENTITY._serialized_start=383 - _FEEDENTITY._serialized_end=593 - _TRIPUPDATE._serialized_start=596 - _TRIPUPDATE._serialized_end=1622 - _TRIPUPDATE_STOPTIMEEVENT._serialized_start=887 - _TRIPUPDATE_STOPTIMEEVENT._serialized_end=968 - _TRIPUPDATE_STOPTIMEUPDATE._serialized_start=971 - _TRIPUPDATE_STOPTIMEUPDATE._serialized_end=1515 - _TRIPUPDATE_STOPTIMEUPDATE_STOPTIMEPROPERTIES._serialized_start=1355 - _TRIPUPDATE_STOPTIMEUPDATE_STOPTIMEPROPERTIES._serialized_end=1417 - _TRIPUPDATE_STOPTIMEUPDATE_SCHEDULERELATIONSHIP._serialized_start=1419 - _TRIPUPDATE_STOPTIMEUPDATE_SCHEDULERELATIONSHIP._serialized_end=1499 - _TRIPUPDATE_TRIPPROPERTIES._serialized_start=1517 - _TRIPUPDATE_TRIPPROPERTIES._serialized_end=1606 - _VEHICLEPOSITION._serialized_start=1625 - _VEHICLEPOSITION._serialized_end=2872 - _VEHICLEPOSITION_CARRIAGEDETAILS._serialized_start=2219 - _VEHICLEPOSITION_CARRIAGEDETAILS._serialized_end=2436 - _VEHICLEPOSITION_VEHICLESTOPSTATUS._serialized_start=2438 - _VEHICLEPOSITION_VEHICLESTOPSTATUS._serialized_end=2509 - _VEHICLEPOSITION_CONGESTIONLEVEL._serialized_start=2511 - _VEHICLEPOSITION_CONGESTIONLEVEL._serialized_end=2636 - _VEHICLEPOSITION_OCCUPANCYSTATUS._serialized_start=2639 - _VEHICLEPOSITION_OCCUPANCYSTATUS._serialized_end=2856 - _ALERT._serialized_start=2875 - _ALERT._serialized_end=4027 - _ALERT_CAUSE._serialized_start=3497 - _ALERT_CAUSE._serialized_end=3713 - _ALERT_EFFECT._serialized_start=3716 - _ALERT_EFFECT._serialized_end=3937 - _ALERT_SEVERITYLEVEL._serialized_start=3939 - _ALERT_SEVERITYLEVEL._serialized_end=4011 - _TIMERANGE._serialized_start=4029 - _TIMERANGE._serialized_end=4084 - _POSITION._serialized_start=4086 - _POSITION._serialized_end=4199 - _TRIPDESCRIPTOR._serialized_start=4202 - _TRIPDESCRIPTOR._serialized_end=4535 - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP._serialized_start=4403 - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP._serialized_end=4519 - _VEHICLEDESCRIPTOR._serialized_start=4537 - _VEHICLEDESCRIPTOR._serialized_end=4622 - _ENTITYSELECTOR._serialized_start=4625 - _ENTITYSELECTOR._serialized_end=4801 - _TRANSLATEDSTRING._serialized_start=4804 - _TRANSLATEDSTRING._serialized_end=4970 - _TRANSLATEDSTRING_TRANSLATION._serialized_start=4893 - _TRANSLATEDSTRING_TRANSLATION._serialized_end=4954 -# @@protoc_insertion_point(module_scope) diff --git a/amarillo/services/gtfsrt/realtime_extension_pb2.py b/amarillo/services/gtfsrt/realtime_extension_pb2.py deleted file mode 100644 index 5db1fda..0000000 --- a/amarillo/services/gtfsrt/realtime_extension_pb2.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: realtime_extension.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -import amarillo.services.gtfsrt.gtfs_realtime_pb2 as gtfs__realtime__pb2 - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18realtime_extension.proto\x12\x10transit_realtime\x1a\x13gtfs-realtime.proto\"p\n\x1bMfdzTripDescriptorExtension\x12\x11\n\troute_url\x18\x01 \x01(\t\x12\x11\n\tagency_id\x18\x02 \x01(\t\x12\x17\n\x0froute_long_name\x18\x03 \x01(\t\x12\x12\n\nroute_type\x18\x04 \x01(\r\"\xb0\x02\n\x1fMfdzStopTimePropertiesExtension\x12X\n\x0bpickup_type\x18\x01 \x01(\x0e\x32\x43.transit_realtime.MfdzStopTimePropertiesExtension.DropOffPickupType\x12Y\n\x0c\x64ropoff_type\x18\x02 \x01(\x0e\x32\x43.transit_realtime.MfdzStopTimePropertiesExtension.DropOffPickupType\"X\n\x11\x44ropOffPickupType\x12\x0b\n\x07REGULAR\x10\x00\x12\x08\n\x04NONE\x10\x01\x12\x10\n\x0cPHONE_AGENCY\x10\x02\x12\x1a\n\x16\x43OORDINATE_WITH_DRIVER\x10\x03:i\n\x0ftrip_descriptor\x12 .transit_realtime.TripDescriptor\x18\xf5\x07 \x01(\x0b\x32-.transit_realtime.MfdzTripDescriptorExtension:\x90\x01\n\x14stop_time_properties\x12>.transit_realtime.TripUpdate.StopTimeUpdate.StopTimeProperties\x18\xf5\x07 \x01(\x0b\x32\x31.transit_realtime.MfdzStopTimePropertiesExtensionB\t\n\x07\x64\x65.mfdz') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'realtime_extension_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - gtfs__realtime__pb2.TripDescriptor.RegisterExtension(trip_descriptor) - gtfs__realtime__pb2.TripUpdate.StopTimeUpdate.StopTimeProperties.RegisterExtension(stop_time_properties) - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\007de.mfdz' - _MFDZTRIPDESCRIPTOREXTENSION._serialized_start=67 - _MFDZTRIPDESCRIPTOREXTENSION._serialized_end=179 - _MFDZSTOPTIMEPROPERTIESEXTENSION._serialized_start=182 - _MFDZSTOPTIMEPROPERTIESEXTENSION._serialized_end=486 - _MFDZSTOPTIMEPROPERTIESEXTENSION_DROPOFFPICKUPTYPE._serialized_start=398 - _MFDZSTOPTIMEPROPERTIESEXTENSION_DROPOFFPICKUPTYPE._serialized_end=486 -# @@protoc_insertion_point(module_scope) diff --git a/amarillo/services/hooks.py b/amarillo/services/hooks.py new file mode 100644 index 0000000..5713585 --- /dev/null +++ b/amarillo/services/hooks.py @@ -0,0 +1,27 @@ +from typing import List +from amarillo.models.Carpool import Carpool + +class CarpoolEvents: + def on_create(cp : Carpool): + pass + def on_update(cp : Carpool): + pass + def on_delete(cp : Carpool): + pass + +carpool_event_listeners : List[CarpoolEvents] = [] + +def register_carpool_event_listener(cpe : CarpoolEvents): + carpool_event_listeners.append(cpe) + +def run_on_create(cp: Carpool): + for cpe in carpool_event_listeners: + cpe.on_create(cp) + +def run_on_update(cp: Carpool): + for cpe in carpool_event_listeners: + cpe.on_update(cp) + +def run_on_delete(cp: Carpool): + for cpe in carpool_event_listeners: + cpe.on_delete(cp) \ No newline at end of file diff --git a/amarillo/services/importing/ride2go.py b/amarillo/services/importing/ride2go.py index 098411c..187ef26 100644 --- a/amarillo/services/importing/ride2go.py +++ b/amarillo/services/importing/ride2go.py @@ -29,7 +29,7 @@ def _extract_carpool(dict) -> Carpool: id=id, agency=agency, deeplink=dict['deeplink'], - stops=[self._extract_stop(s) for s in dict.get('stops')], + stops=[Ride2GoImporter._extract_stop(s) for s in dict.get('stops')], departureTime=dict.get('departTime'), departureDate=dict.get('departDate') if dict.get('departDate') else dict.get('weekdays'), lastUpdated=dict.get('lastUpdated'), diff --git a/amarillo/services/regions.py b/amarillo/services/regions.py index 8be79e0..425b3ac 100644 --- a/amarillo/services/regions.py +++ b/amarillo/services/regions.py @@ -9,8 +9,7 @@ class RegionService: def __init__(self): self.regions: Dict[str, Region] = {} - - for region_file_name in glob('conf/region/*.json'): + for region_file_name in glob('data/region/*.json'): with open(region_file_name) as region_file: dict = json.load(region_file) region = Region(**dict) diff --git a/amarillo/services/routing.py b/amarillo/services/routing.py deleted file mode 100644 index 96f6229..0000000 --- a/amarillo/services/routing.py +++ /dev/null @@ -1,47 +0,0 @@ -import requests -import logging - -logger = logging.getLogger(__name__) - -class RoutingException(Exception): - def __init__(self, message): - # Call Exception.__init__(message) - # to use the same Message header as the parent class - super().__init__(message) - -class RoutingService(): - def __init__(self, gh_url = 'https://api.mfdz.de/gh'): - self.gh_service_url = gh_url - - def path_for_stops(self, points): - # Retrieve graphhopper route traversing given points - directions = self._get_directions(points) - if directions and len(directions.get("paths"))>0: - return directions.get("paths")[0] - else: - return {} - - def _get_directions(self, points): - req_url = self._create_url(points, True, True) - logger.debug("Get directions via: {}".format(req_url)) - response = requests.get(req_url) - status = response.status_code - if status == 200: - # Found route between points - return response.json() - else: - try: - message = response.json().get('message') - except: - raise RoutingException("Get directions failed with status code {}".format(status)) - else: - raise RoutingException(message) - - def _create_url(self, points, calc_points = False, instructions = False): - """ Creates GH request URL """ - locations = "" - for point in points: - locations += "point={0}%2C{1}&".format(point.y, point.x) - - return "{0}/route?{1}instructions={2}&calc_points={3}&points_encoded=false&profile=car".format( - self.gh_service_url, locations, instructions, calc_points) diff --git a/amarillo/services/secrets.py b/amarillo/services/secrets.py index 1afd93d..773039e 100644 --- a/amarillo/services/secrets.py +++ b/amarillo/services/secrets.py @@ -1,9 +1,8 @@ -from typing import Dict -from pydantic import Field +from pydantic import Field, ConfigDict from pydantic_settings import BaseSettings - # Example: secrets = { "mfdz": "some secret" } class Secrets(BaseSettings): + model_config = ConfigDict(extra='allow') # Allow plugins to add extra values ride2go_token: str = Field(None, env = 'RIDE2GO_TOKEN') diff --git a/amarillo/services/stop_importer/__init__.py b/amarillo/services/stop_importer/__init__.py deleted file mode 100644 index 0c9769a..0000000 --- a/amarillo/services/stop_importer/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from .base import ( - CsvStopsImporter as CsvStopsImporter, -) -from .base import ( - GeojsonStopsImporter as GeojsonStopsImporter, -) -from .gtfs import GtfsStopsImporter as GtfsStopsImporter -from .overpass import OverpassStopsImporter as OverpassStopsImporter diff --git a/amarillo/services/stop_importer/base.py b/amarillo/services/stop_importer/base.py deleted file mode 100644 index b80c99b..0000000 --- a/amarillo/services/stop_importer/base.py +++ /dev/null @@ -1,79 +0,0 @@ -import codecs -import csv -import logging -import re - -import geopandas as gpd -import requests - -logger = logging.getLogger(__name__) - - -class BaseStopsImporter: - def _normalize_stop_name(self, stop_name): - # if the name is empty, we set P+R as a fall back. However, it should be named at the source - default_name = 'P+R' - if stop_name in ('', 'Park&Ride'): - return default_name - return re.sub(r'P(ark)?\s?[\+&]\s?R(ail|ide)?', 'P+R', stop_name) - - def _as_dataframe(self, id, lat, lon, stop_name): - df = gpd.GeoDataFrame(data={'x': lon, 'y': lat, 'stop_name': stop_name, 'id': id}) - return gpd.GeoDataFrame(df, geometry=gpd.points_from_xy(df.x, df.y, crs='EPSG:4326')) - - -class CsvStopsImporter(BaseStopsImporter): - DEFAULT_COLUMN_MAPPING = { - 'stop_id': 'stop_id', - 'stop_lat': 'stop_lat', - 'stop_lon': 'stop_lon', - 'stop_name': 'stop_name', - } - - def load_stops(self, source, timeout=15): - if source.startswith('http'): - with requests.get(source, timeout=timeout) as csv_source: - return self._load_stops_from_csv_source( - codecs.iterdecode(csv_source.iter_lines(), 'utf-8'), delimiter=';' - ) - else: - with open(source, encoding='utf-8') as csv_source: - return self._load_stops_from_csv_source(csv_source, delimiter=';') - - def _load_stops_from_csv_source(self, csv_source, delimiter: str = ',', column_mapping=None): - if column_mapping is None: - column_mapping = self.DEFAULT_COLUMN_MAPPING - id = [] - lat = [] - lon = [] - stop_name = [] - reader = csv.DictReader(csv_source, delimiter=delimiter) - for row in reader: - id.append(row[column_mapping['stop_id']]) - lat.append(float(row[column_mapping['stop_lat']].replace(',', '.'))) - lon.append(float(row[column_mapping['stop_lon']].replace(',', '.'))) - stop_name.append(self._normalize_stop_name(row[column_mapping['stop_name']])) - - return self._as_dataframe(id, lat, lon, stop_name) - - -class GeojsonStopsImporter(BaseStopsImporter): - def load_stops(self, source, timeout=15): - with requests.get(source, timeout=timeout) as json_source: - geojson_source = json_source.json() - id = [] - lat = [] - lon = [] - stop_name = [] - for row in geojson_source['features']: - coord = row['geometry']['coordinates'] - if not coord or not row['properties'].get('name'): - logger.error('Stop feature {} has null coord or name'.format(row['id'])) - continue - - id.append(row['id']) - lon.append(coord[0]) - lat.append(coord[1]) - stop_name.append(self._normalize_stop_name(row['properties']['name'])) - - return self._as_dataframe(id, lat, lon, stop_name) diff --git a/amarillo/services/stop_importer/gtfs.py b/amarillo/services/stop_importer/gtfs.py deleted file mode 100644 index b895b5f..0000000 --- a/amarillo/services/stop_importer/gtfs.py +++ /dev/null @@ -1,29 +0,0 @@ -import io -import zipfile -from pathlib import Path - -import requests - -from .base import CsvStopsImporter - - -class GtfsStopsImporter(CsvStopsImporter): - def load_stops(self, id, url, timeout=15, **kwargs): - if url.startswith('http'): - # TODO: only reload if file is older than x - gtfs_file = Path(f'data/{id}.gtfs.zip') - with requests.get(url, timeout=timeout) as response: - if response.ok: - self._store_response(gtfs_file, response) - else: - gtfs_file = url - - with zipfile.ZipFile(gtfs_file) as gtfs: - with gtfs.open('stops.txt', 'r') as stops_file: - return self._load_stops_from_csv_source(io.TextIOWrapper(stops_file, 'utf-8-sig')) - - def _store_response(self, filename, response): - with filename.open('wb') as file: - for chunk in response.iter_content(chunk_size=1024 * 1024): - if chunk: - file.write(chunk) diff --git a/amarillo/services/stop_importer/overpass.py b/amarillo/services/stop_importer/overpass.py deleted file mode 100644 index 8067471..0000000 --- a/amarillo/services/stop_importer/overpass.py +++ /dev/null @@ -1,39 +0,0 @@ -import csv -import io -import logging - -import requests - -from .base import BaseStopsImporter - -logger = logging.getLogger(__name__) - - -class OverpassStopsImporter(BaseStopsImporter): - def load_stops(self, area_selector, timeout=15, **kwargs): - query = f''' - [out:csv(::"type", ::"id", ::"lat", ::"lon", name,parking,park_ride,operator,access,lit,fee,capacity,"capacity:disabled",supervised,surface,covered,maxstay,opening_hours)][timeout:60]; - area{area_selector}->.a; - nwr(area.a)[park_ride][park_ride!=no][access!=customers]; - out center; - ''' - - response = requests.post('https://overpass-api.de/api/interpreter', data=query, timeout=timeout) - if not response.ok: - logger.error(f'Error retrieving stops from overpass: {response.text}') - - return self._parse_overpass_csv_response(response.text.splitlines()) - - def _parse_overpass_csv_response(self, csv_source): - id = [] - lat = [] - lon = [] - stop_name = [] - reader = csv.DictReader(csv_source, delimiter='\t') - for row in reader: - id.append(f'osm:{row["@type"][0]}{row["@id"]}') - lat.append(float(row['@lat'])) - lon.append(float(row['@lon'])) - stop_name.append(self._normalize_stop_name(row['name'])) - - return self._as_dataframe(id, lat, lon, stop_name) diff --git a/amarillo/services/stops.py b/amarillo/services/stops.py deleted file mode 100644 index 1c00791..0000000 --- a/amarillo/services/stops.py +++ /dev/null @@ -1,131 +0,0 @@ -import codecs -import csv -import logging -import re -from contextlib import closing -from io import TextIOWrapper - -import geopandas as gpd -import pandas as pd -import requests -from pyproj import Proj, Transformer -from shapely.geometry import LineString, Point -from shapely.ops import transform - -from amarillo.models.Carpool import StopTime - -from .stop_importer import CsvStopsImporter, GeojsonStopsImporter, GtfsStopsImporter, OverpassStopsImporter - -logger = logging.getLogger(__name__) - - -def is_carpooling_stop(stop_id, name): - stop_name = name.lower() - # mfdz: or bbnavi: prefixed stops are custom stops which are explicitly meant to be carpooling stops - return stop_id.startswith('mfdz:') or stop_id.startswith('bbnavi:') or 'mitfahr' in stop_name or 'p&m' in stop_name - - -class StopsStore: - def __init__(self, stop_sources=None, internal_projection='EPSG:32632'): - self.internal_projection = internal_projection - self.projection = Transformer.from_crs('EPSG:4326', internal_projection, always_xy=True).transform - self.stopsDataFrames = [] - self.stop_sources = stop_sources if stop_sources is not None else [] - - def load_stop_sources(self): - """Imports stops from stop_sources and registers them with - the distance they are still associated with a trip. - E.g. bus stops should be registered with a distance of e.g. 30m, - while larger carpool parkings might be registered with e.g. 500m. - - Subsequent calls of load_stop_sources will reload all stop_sources - but replace the current stops only if all stops could be loaded successfully. - """ - stopsDataFrames = [] - error_occured = False - - for stops_source in self.stop_sources: - try: - source_url = stops_source.get('url') - source_type = stops_source.get('type') or ( - 'geojson' - if source_url is not None and source_url.startswith('http') and source_url.endswith('json') - else 'csv' - ) - logger.info('Loading stop source %s...', stops_source.get('id')) - match source_type: - case 'geojson': - stopsDataFrame = GeojsonStopsImporter().load_stops(source_url) - case 'csv': - stopsDataFrame = CsvStopsImporter().load_stops(source_url) - case 'overpass': - stopsDataFrame = OverpassStopsImporter().load_stops(**stops_source) - case 'gtfs': - stopsDataFrame = GtfsStopsImporter().load_stops(**stops_source) - case _: - logger.error('Failed to load stops, source type %s not supported', source_type) - continue - stopsDataFrame.to_crs(crs=self.internal_projection, inplace=True) - stopsDataFrames.append({'distanceInMeter': stops_source['vicinity'], 'stops': stopsDataFrame}) - except Exception: - error_occured = True - logger.error('Failed to load stops from %s to StopsStore.', stops_source, exc_info=True) - - if not error_occured: - self.stopsDataFrames = stopsDataFrames - - def find_additional_stops_around(self, line, stops=None): - """Returns a GeoDataFrame with all stops in vicinity of the - given line, sorted by distance from origin of the line. - Note: for internal projection/distance calculations, the - lat/lon geometries of line and stops are converted to - """ - stops_frames = [] - if stops: - stops_frames.append(self._convert_to_dataframe(stops)) - transformedLine = transform(self.projection, LineString(line.coordinates)) - for stops_to_match in self.stopsDataFrames: - stops_frames.append( - self._find_stops_around_transformed( - stops_to_match['stops'], transformedLine, stops_to_match['distanceInMeter'] - ) - ) - stops = gpd.GeoDataFrame(pd.concat(stops_frames, ignore_index=True, sort=True)) - if not stops.empty: - self._sort_by_distance(stops, transformedLine) - return stops - - def find_closest_stop(self, carpool_stop, max_search_distance): - transformedCoord = Point(self.projection(carpool_stop.lon, carpool_stop.lat)) - best_dist = max_search_distance + 1 - best_stop = None - for stops_with_dist in self.stopsDataFrames: - stops = stops_with_dist['stops'] - s, d = stops.sindex.nearest( - transformedCoord, return_all=True, return_distance=True, max_distance=max_search_distance - ) - if len(d) > 0 and d[0] < best_dist: - best_dist = d[0] - row = s[1][0] - best_stop = StopTime(name=stops.at[row, 'stop_name'], lat=stops.at[row, 'y'], lon=stops.at[row, 'x']) - - return best_stop if best_stop else carpool_stop - - def _find_stops_around_transformed(self, stopsDataFrame, transformedLine, distance): - bufferedLine = transformedLine.buffer(distance) - sindex = stopsDataFrame.sindex - possible_matches_index = list(sindex.intersection(bufferedLine.bounds)) - possible_matches = stopsDataFrame.iloc[possible_matches_index] - - return possible_matches[possible_matches.intersects(bufferedLine)] - - def _convert_to_dataframe(self, stops): - return gpd.GeoDataFrame( - [[stop.name, stop.lon, stop.lat, stop.id, Point(self.projection(stop.lon, stop.lat))] for stop in stops], - columns=['stop_name', 'x', 'y', 'id', 'geometry'], - crs=self.internal_projection, - ) - - def _sort_by_distance(self, stops, transformedLine): - stops['distance'] = stops.apply(lambda row: transformedLine.project(row['geometry']), axis=1) - stops.sort_values('distance', inplace=True) diff --git a/amarillo/services/trips.py b/amarillo/services/trips.py deleted file mode 100644 index fcbc21b..0000000 --- a/amarillo/services/trips.py +++ /dev/null @@ -1,375 +0,0 @@ -from amarillo.services.config import config -from amarillo.models.gtfs import GtfsTimeDelta, GtfsStopTime -from amarillo.models.Carpool import MAX_STOPS_PER_TRIP, Carpool, Weekday, StopTime, PickupDropoffType -from amarillo.services.gtfs_constants import * -from amarillo.services.routing import RoutingService, RoutingException -from amarillo.services.stops import is_carpooling_stop -from amarillo.utils.utils import assert_folder_exists, is_older_than_days, yesterday, geodesic_distance_in_m -from shapely.geometry import Point, LineString, box -from geojson_pydantic.geometries import LineString as GeoJSONLineString -from datetime import datetime, timedelta -import numpy as np -import os -import json -import logging - -logger = logging.getLogger(__name__) - -class Trip: - - def __init__(self, trip_id, route_name, headsign, url, calendar, departureTime, path, agency, lastUpdated, stop_times, bbox): - if isinstance(calendar, set): - self.runs_regularly = True - self.weekdays = [ - 1 if Weekday.monday in calendar else 0, - 1 if Weekday.tuesday in calendar else 0, - 1 if Weekday.wednesday in calendar else 0, - 1 if Weekday.thursday in calendar else 0, - 1 if Weekday.friday in calendar else 0, - 1 if Weekday.saturday in calendar else 0, - 1 if Weekday.sunday in calendar else 0, - ] - start_in_day = self._total_seconds(departureTime) - else: - self.start = datetime.combine(calendar, departureTime) - self.runs_regularly = False - self.weekdays = [0,0,0,0,0,0,0] - - self.start_time = departureTime - self.path = path - self.trip_id = trip_id - self.url = url - self.agency = agency - self.stops = [] - self.lastUpdated = lastUpdated - self.stop_times = stop_times - self.bbox = bbox - self.route_name = route_name - self.trip_headsign = headsign - - def path_as_line_string(self): - return path - - def _total_seconds(self, instant): - return instant.hour * 3600 + instant.minute * 60 + instant.second - - def start_time_str(self): - return self.start_time.strftime("%H:%M:%S") - - def next_trip_dates(self, start_date, day_count=14): - if self.runs_regularly: - for single_date in (start_date + timedelta(n) for n in range(day_count)): - if self.weekdays[single_date.weekday()]==1: - yield single_date.strftime("%Y%m%d") - else: - yield self.start.strftime("%Y%m%d") - - def route_long_name(self): - return self.route_name - - def intersects(self, bbox): - return self.bbox.intersects(box(*bbox)) - - -class TripStore(): - """ - TripStore maintains the currently valid trips. A trip is a - carpool offer enhanced with all stops this - - Attributes: - trips Dict of currently valid trips. - deleted_trips Dict of recently deleted trips. - """ - - def __init__(self, stops_store): - self.transformer = TripTransformer(stops_store) - self.stops_store = stops_store - self.trips = {} - self.deleted_trips = {} - self.recent_trips = {} - - - def put_carpool(self, carpool: Carpool): - """ - Adds carpool to the TripStore. - """ - id = "{}:{}".format(carpool.agency, carpool.id) - filename = f'data/enhanced/{carpool.agency}/{carpool.id}.json' - try: - existing_carpool = self._load_carpool_if_exists(carpool.agency, carpool.id) - if existing_carpool and existing_carpool.lastUpdated == carpool.lastUpdated: - enhanced_carpool = existing_carpool - else: - if len(carpool.stops) < 2 or self.distance_in_m(carpool) < 1000: - logger.warning("Failed to add carpool %s:%s to TripStore, distance too low", carpool.agency, carpool.id) - self.handle_failed_carpool_enhancement(carpool) - return - enhanced_carpool = self.transformer.enhance_carpool(carpool) - # TODO should only store enhanced_carpool, if it has 2 or more stops - assert_folder_exists(f'data/enhanced/{carpool.agency}/') - with open(filename, 'w', encoding='utf-8') as f: - f.write(enhanced_carpool.json()) - logger.info("Added enhanced carpool %s:%s", carpool.agency, carpool.id) - - return self._load_as_trip(enhanced_carpool) - except RoutingException as err: - logger.warning("Failed to add carpool %s:%s to TripStore due to RoutingException %s", carpool.agency, carpool.id, getattr(err, 'message', repr(err))) - self.handle_failed_carpool_enhancement(carpool) - except Exception as err: - logger.error("Failed to add carpool %s:%s to TripStore.", carpool.agency, carpool.id, exc_info=True) - self.handle_failed_carpool_enhancement(carpool) - - def handle_failed_carpool_enhancement(sellf, carpool: Carpool): - assert_folder_exists(f'data/failed/{carpool.agency}/') - with open(f'data/failed/{carpool.agency}/{carpool.id}.json', 'w', encoding='utf-8') as f: - f.write(carpool.json()) - - def distance_in_m(self, carpool): - if len(carpool.stops) < 2: - return 0 - s1 = carpool.stops[0] - s2 = carpool.stops[-1] - return geodesic_distance_in_m((s1.lon, s1.lat),(s2.lon, s2.lat)) - - def recently_added_trips(self): - return list(self.recent_trips.values()) - - def recently_deleted_trips(self): - return list(self.deleted_trips.values()) - - def _load_carpool_if_exists(self, agency_id: str, carpool_id: str): - if carpool_exists(agency_id, carpool_id, 'data/enhanced'): - try: - return load_carpool(agency_id, carpool_id, 'data/enhanced') - except Exception as e: - # An error on restore could be caused by model changes, - # in such a case, it need's to be recreated - logger.warning("Could not restore enhanced trip %s:%s, reason: %s", agency_id, carpool_id, repr(e)) - - return None - - def _load_as_trip(self, carpool: Carpool): - trip = self.transformer.transform_to_trip(carpool) - id = trip.trip_id - self.trips[id] = trip - if not is_older_than_days(carpool.lastUpdated, 1): - self.recent_trips[id] = trip - logger.debug("Added trip %s", id) - - return trip - - def delete_carpool(self, agency_id: str, carpool_id: str): - """ - Deletes carpool from the TripStore. - """ - agencyScopedCarpoolId = f"{agency_id}:{carpool_id}" - trip_to_be_deleted = self.trips.get(agencyScopedCarpoolId) - if trip_to_be_deleted: - self.deleted_trips[agencyScopedCarpoolId] = trip_to_be_deleted - del self.trips[agencyScopedCarpoolId] - - if self.recent_trips.get(agencyScopedCarpoolId): - del self.recent_trips[agencyScopedCarpoolId] - - if carpool_exists(agency_id, carpool_id): - remove_carpool_file(agency_id, carpool_id) - - logger.debug("Deleted trip %s", id) - - def unflag_unrecent_updates(self): - """ - Trips that were last updated before yesterday, are not recent - any longer. As no updates need to be sent for them any longer, - they will be removed from recent recent_trips and deleted_trips. - """ - for key in list(self.recent_trips): - t = self.recent_trips.get(key) - if t and t.lastUpdated.date() < yesterday(): - del self.recent_trips[key] - - for key in list(self.deleted_trips): - t = self.deleted_trips.get(key) - if t and t.lastUpdated.date() < yesterday(): - del self.deleted_trips[key] - - -class TripTransformer: - REPLACE_CARPOOL_STOPS_BY_CLOSEST_TRANSIT_STOPS = True - REPLACEMENT_STOPS_SERACH_RADIUS_IN_M = 1000 - SIMPLIFY_TOLERANCE = 0.0001 - - router = RoutingService(config.graphhopper_base_url) - - def __init__(self, stops_store): - self.stops_store = stops_store - - def transform_to_trip(self, carpool): - stop_times = self._convert_stop_times(carpool) - route_name = carpool.stops[0].name + " nach " + carpool.stops[-1].name - headsign= carpool.stops[-1].name - trip_id = self._trip_id(carpool) - path = carpool.path - bbox = box( - min([pt[0] for pt in path.coordinates]), - min([pt[1] for pt in path.coordinates]), - max([pt[0] for pt in path.coordinates]), - max([pt[1] for pt in path.coordinates])) - - trip = Trip(trip_id, route_name, headsign, str(carpool.deeplink), carpool.departureDate, carpool.departureTime, carpool.path, carpool.agency, carpool.lastUpdated, stop_times, bbox) - - return trip - - def _trip_id(self, carpool): - return f"{carpool.agency}:{carpool.id}" - - def _replace_stops_by_transit_stops(self, carpool, max_search_distance): - new_stops = [] - for carpool_stop in carpool.stops: - new_stops.append(self.stops_store.find_closest_stop(carpool_stop, max_search_distance)) - return new_stops - - def enhance_carpool(self, carpool): - if self.REPLACE_CARPOOL_STOPS_BY_CLOSEST_TRANSIT_STOPS: - carpool.stops = self._replace_stops_by_transit_stops(carpool, self.REPLACEMENT_STOPS_SERACH_RADIUS_IN_M) - - path = self._path_for_ride(carpool) - lineString_shapely_wgs84 = LineString(coordinates = path["points"]["coordinates"]).simplify(0.0001) - lineString_wgs84 = GeoJSONLineString(type="LineString", coordinates=list(lineString_shapely_wgs84.coords)) - virtual_stops = self.stops_store.find_additional_stops_around(lineString_wgs84, carpool.stops) - if not virtual_stops.empty: - virtual_stops["time"] = self._estimate_times(path, virtual_stops['distance']) - logger.debug("Virtual stops found: {}".format(virtual_stops)) - if len(virtual_stops) > MAX_STOPS_PER_TRIP: - # in case we found more than MAX_STOPS_PER_TRIP, we retain first and last - # half of MAX_STOPS_PER_TRIP - virtual_stops = virtual_stops.iloc[np.r_[0:int(MAX_STOPS_PER_TRIP/2), int(MAX_STOPS_PER_TRIP/2):]] - - trip_id = f"{carpool.agency}:{carpool.id}" - stop_times = self._stops_and_stop_times(carpool.departureTime, trip_id, virtual_stops) - - enhanced_carpool = carpool.copy() - enhanced_carpool.stops = stop_times - enhanced_carpool.path = lineString_wgs84 - return enhanced_carpool - - def _convert_stop_times(self, carpool): - - stop_times = [GtfsStopTime( - self._trip_id(carpool), - stop.arrivalTime, - stop.departureTime, - stop.id, - seq_nr+1, - STOP_TIMES_STOP_TYPE_NONE if stop.pickup_dropoff == PickupDropoffType.only_dropoff else STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER, - STOP_TIMES_STOP_TYPE_NONE if stop.pickup_dropoff == PickupDropoffType.only_pickup else STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER, - STOP_TIMES_TIMEPOINT_APPROXIMATE) - for seq_nr, stop in enumerate(carpool.stops)] - return stop_times - - def _path_for_ride(self, carpool): - points = self._stop_coords(carpool.stops) - return self.router.path_for_stops(points) - - def _stop_coords(self, stops): - # Retrieve coordinates of all officially announced stops (start, intermediate, target) - return [Point(stop.lon, stop.lat) for stop in stops] - - def _estimate_times(self, path, distances_from_start): - cumulated_distance = 0 - cumulated_time = 0 - stop_times = [] - instructions = path["instructions"] - - cnt = 0 - instr_distance = instructions[cnt]["distance"] - instr_time = instructions[cnt]["time"] - - for distance in distances_from_start: - while cnt < len(instructions) and cumulated_distance + instructions[cnt]["distance"] < distance: - cumulated_distance = cumulated_distance + instructions[cnt]["distance"] - cumulated_time = cumulated_time + instructions[cnt]["time"] - cnt = cnt + 1 - - if cnt < len(instructions): - if instructions[cnt]["distance"] ==0: - raise RoutingException("Origin and destinaction too close") - percent_dist = (distance - cumulated_distance) / instructions[cnt]["distance"] - stop_time = cumulated_time + percent_dist * instructions[cnt]["time"] - stop_times.append(stop_time) - else: - logger.debug("distance {} exceeds total length {}, using max arrival time {}".format(distance, cumulated_distance, cumulated_time)) - stop_times.append(cumulated_time) - return stop_times - - def _stops_and_stop_times(self, start_time, trip_id, stops_frame): - # Assumptions: - # arrival_time = departure_time - # pickup_type, drop_off_type for origin: = coordinate/none - # pickup_type, drop_off_type for destination: = none/coordinate - # timepoint = approximate for origin and destination (not sure what consequences this might have for trip planners) - number_of_stops = len(stops_frame.index) - total_distance = stops_frame.iloc[number_of_stops-1]["distance"] - - first_stop_time = GtfsTimeDelta(hours = start_time.hour, minutes = start_time.minute, seconds = start_time.second) - stop_times = [] - seq_nr = 0 - for i in range(0, number_of_stops): - current_stop = stops_frame.iloc[i] - - if not current_stop.id: - continue - elif i == 0: - if (stops_frame.iloc[1].time-current_stop.time) < 1000: - # skip custom stop if there is an official stop very close by - logger.debug("Skipped stop %s", current_stop.id) - continue - else: - if (current_stop.time-stops_frame.iloc[i-1].time) < 5000 and not i==1 and not is_carpooling_stop(current_stop.id, current_stop.stop_name): - # skip latter stop if it's very close (<5 seconds drive) by the preceding - logger.debug("Skipped stop %s", current_stop.id) - continue - trip_time = timedelta(milliseconds=int(current_stop.time)) - is_dropoff = self._is_dropoff_stop(current_stop, total_distance) - is_pickup = self._is_pickup_stop(current_stop, total_distance) - # TODO would be nice if possible to publish a minimum shared distance - pickup_type = STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER if is_pickup else STOP_TIMES_STOP_TYPE_NONE - dropoff_type = STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER if is_dropoff else STOP_TIMES_STOP_TYPE_NONE - - if is_pickup and not is_dropoff: - pickup_dropoff = PickupDropoffType.only_pickup - elif not is_pickup and is_dropoff: - pickup_dropoff = PickupDropoffType.only_dropoff - else: - pickup_dropoff = PickupDropoffType.pickup_and_dropoff - - next_stop_time = first_stop_time + trip_time - seq_nr += 1 - stop_times.append(StopTime(**{ - 'arrivalTime': str(next_stop_time), - 'departureTime': str(next_stop_time), - 'id': current_stop.id, - 'pickup_dropoff': pickup_dropoff, - 'name': str(current_stop.stop_name), - 'lat': current_stop.y, - 'lon': current_stop.x - })) - - return stop_times - - def _is_dropoff_stop(self, current_stop, total_distance): - return current_stop["distance"] >= 0.5 * total_distance - - def _is_pickup_stop(self, current_stop, total_distance): - return current_stop["distance"] < 0.5 * total_distance - -def load_carpool(agency_id: str, carpool_id: str, folder: str ='data/enhanced') -> Carpool: - with open(f'{folder}/{agency_id}/{carpool_id}.json', 'r', encoding='utf-8') as f: - dict = json.load(f) - carpool = Carpool(**dict) - return carpool - -def carpool_exists(agency_id: str, carpool_id: str, folder: str ='data/enhanced'): - return os.path.exists(f"{folder}/{agency_id}/{carpool_id}.json") - -def remove_carpool_file(agency_id: str, carpool_id: str, folder: str ='data/enhanced'): - return os.remove(f"{folder}/{agency_id}/{carpool_id}.json") diff --git a/config b/amarillo/static/config similarity index 100% rename from config rename to amarillo/static/config diff --git a/conf/agency/mfdz.json b/amarillo/static/data/agency/mfdz.json similarity index 100% rename from conf/agency/mfdz.json rename to amarillo/static/data/agency/mfdz.json diff --git a/conf/agency/mifaz.json b/amarillo/static/data/agency/mifaz.json similarity index 100% rename from conf/agency/mifaz.json rename to amarillo/static/data/agency/mifaz.json diff --git a/conf/agency/ride2go.json b/amarillo/static/data/agency/ride2go.json similarity index 100% rename from conf/agency/ride2go.json rename to amarillo/static/data/agency/ride2go.json diff --git a/conf/agency/ummadum.json b/amarillo/static/data/agency/ummadum.json similarity index 100% rename from conf/agency/ummadum.json rename to amarillo/static/data/agency/ummadum.json diff --git a/conf/region/bb.json b/amarillo/static/data/region/bb.json similarity index 100% rename from conf/region/bb.json rename to amarillo/static/data/region/bb.json diff --git a/conf/region/bw.json b/amarillo/static/data/region/bw.json similarity index 100% rename from conf/region/bw.json rename to amarillo/static/data/region/bw.json diff --git a/conf/region/by.json b/amarillo/static/data/region/by.json similarity index 100% rename from conf/region/by.json rename to amarillo/static/data/region/by.json diff --git a/conf/region/nrw.json b/amarillo/static/data/region/nrw.json similarity index 100% rename from conf/region/nrw.json rename to amarillo/static/data/region/nrw.json diff --git a/conf/stop_sources.json b/amarillo/static/data/stop_sources.json similarity index 100% rename from conf/stop_sources.json rename to amarillo/static/data/stop_sources.json diff --git a/amarillo/static/logging.conf b/amarillo/static/logging.conf new file mode 100644 index 0000000..142287a --- /dev/null +++ b/amarillo/static/logging.conf @@ -0,0 +1,28 @@ +[loggers] +keys=root + +[handlers] +keys=consoleHandler, fileHandler + +[formatters] +keys=simpleFormatter + +[logger_root] +level=INFO +handlers=consoleHandler, fileHandler +propagate=yes + +[handler_consoleHandler] +class=StreamHandler +level=DEBUG +formatter=simpleFormatter +args=(sys.stdout,) + +[handler_fileHandler] +class=handlers.RotatingFileHandler +level=ERROR +formatter=simpleFormatter +args=('error.log', 'a', 1000000, 3) # Filename, mode, maxBytes, backupCount + +[formatter_simpleFormatter] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s diff --git a/static/css/docs.css b/amarillo/static/static/css/docs.css similarity index 100% rename from static/css/docs.css rename to amarillo/static/static/css/docs.css diff --git a/static/css/theme.css b/amarillo/static/static/css/theme.css similarity index 100% rename from static/css/theme.css rename to amarillo/static/static/css/theme.css diff --git a/static/img/cloud.png b/amarillo/static/static/img/cloud.png similarity index 100% rename from static/img/cloud.png rename to amarillo/static/static/img/cloud.png diff --git a/static/img/favicon.ico b/amarillo/static/static/img/favicon.ico similarity index 100% rename from static/img/favicon.ico rename to amarillo/static/static/img/favicon.ico diff --git a/templates/home/index.html b/amarillo/static/templates/home/index.html similarity index 100% rename from templates/home/index.html rename to amarillo/static/templates/home/index.html diff --git a/templates/shared/layout.html b/amarillo/static/templates/shared/layout.html similarity index 100% rename from templates/shared/layout.html rename to amarillo/static/templates/shared/layout.html diff --git a/amarillo/tests/fixtures/stops.gtfs.zip b/amarillo/tests/fixtures/stops.gtfs.zip deleted file mode 100644 index a1a2d89..0000000 Binary files a/amarillo/tests/fixtures/stops.gtfs.zip and /dev/null differ diff --git a/amarillo/tests/fixtures/stops_overpass_result.csv b/amarillo/tests/fixtures/stops_overpass_result.csv deleted file mode 100644 index 80a22df..0000000 --- a/amarillo/tests/fixtures/stops_overpass_result.csv +++ /dev/null @@ -1,40 +0,0 @@ -@type @id @lat @lon name parking park_ride operator access lit fee capacity capacity:disabled supervised surface covered maxstay opening_hours -node 5206558994 46.7470278 11.7621206 underground yes Gemeinde Lüsen - Comune di Luson yes no 150 yes asphalt -way 32003348 46.3187380 11.2571309 surface train no -way 32065524 46.4731774 11.3366206 surface bus no yes asphalt 12 hours -way 41533333 46.8946679 11.4416069 surface train -way 41761911 46.6194999 10.8595329 surface train yes no asphalt -way 42756580 46.7977285 11.8437006 surface train -way 49516469 46.7129317 11.6502813 Zugbahnhof surface train yes -way 103443671 46.9374689 11.4421890 surface train yes -way 105577704 46.7977222 11.6699031 surface train yes no 40 3 -way 118054506 46.2457058 11.2033435 surface train -way 118117379 46.2464468 11.2050366 surface train -way 118117771 46.2460096 11.2038988 surface train -way 125646510 46.3624712 11.2981399 surface train 350 -way 163831940 46.5153270 11.3584047 surface yes permissive no 30 no asphalt 3 hours -way 163944040 46.8695883 11.4875907 surface train -way 205943588 46.5632720 11.2146666 yes -way 205967625 46.5580712 11.2233703 surface yes 8 -way 216262561 46.7344967 12.2786041 surface yes Gemeinde Innichen yes no -way 216262563 46.7331734 12.2740601 surface yes Gemeinde Innichen yes yes no -way 216271211 46.7316730 12.2742284 Parcheggio Ovest - Parkplatz West surface yes Comune di San Candido - Gemeinde Innichen yes no no -way 222883790 46.4346658 11.3243974 surface yes yes yes no -way 226233767 46.6839959 10.5475758 Bahnhof Mals - parcheggio Stazione Malles Venosta surface SAD/Postbus yes no 80 -way 244943953 46.4060262 11.3164184 surface yes 4 -way 306160473 46.6194129 10.8607646 surface train no ground -way 365690132 46.8964531 11.4384408 surface train Gemeinde Comune yes yes 40 -way 374138741 46.6433335 11.5740285 surface train no -way 490868318 46.4347608 11.3240262 street_side yes yes yes no -way 490868319 46.4348386 11.3236228 street_side train no yes -way 497761617 46.6475928 11.6171125 surface bus yes no -way 580769922 46.5558885 11.7659849 surface yes yes ground 1 day winter -way 588469607 46.7113808 10.5334902 surface bus yes no 4 asphalt -way 710630129 46.6422784 11.5741234 street_side yes yes no 36 -way 711268816 46.3188415 11.2570029 surface train no -way 711268817 46.3186680 11.2572697 surface train no -way 711325961 46.6017848 11.5333487 surface train no -way 939778385 46.7982901 11.6702231 surface train -way 939778387 46.7967968 11.6695732 surface train yes no -way 945996879 46.6421469 11.5741285 yes 12 -way 1119115639 46.7203354 10.8628890 surface bus diff --git a/amarillo/tests/stops.csv b/amarillo/tests/stops.csv deleted file mode 100644 index ed419bb..0000000 --- a/amarillo/tests/stops.csv +++ /dev/null @@ -1,5 +0,0 @@ -stop_id;stop_code;stop_lat;stop_lon;stop_name -mfdz:x;x;52.11901;14.2;Stop x -mfdz:y;y;53.1;14.01;Stop y -mfdz:z;z;54.11;14.0;Stop z -mfdz:Ang001;Ang001;53.11901;14.015776;Mitfahrbank Biesenbrow diff --git a/amarillo/tests/stops.json b/amarillo/tests/stops.json deleted file mode 100644 index 5744c13..0000000 --- a/amarillo/tests/stops.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "data": { - "pointsOfInterest": [ - { - "id": "14622", - "externalId": "bbnavi:12073:0001", - "name": "Parkbank", - "description": "Parkbank", - "dataProvider": { - "id": "1", - "name": "Administrator" - }, - "addresses": [ - { - "street": "Hauptstrasse", - "city": "Wittenberge", - "zip": "12345", - "geoLocation": { - "latitude": 52.9932971109789, - "longitude": 11.767383582547 - } - } - ], - "openStreetMap": { - "capacity": 112, - "capacityCharging": "2", - "capacityDisabled": "", - "fee": "No", - "lit": "Yes", - "parking": "", - "shelter": "No", - "surface": "", - "utilization": "", - "website": "" - } - } - ] - } -} \ No newline at end of file diff --git a/amarillo/tests/test_gtfs.py b/amarillo/tests/test_gtfs.py deleted file mode 100644 index 61a8e5a..0000000 --- a/amarillo/tests/test_gtfs.py +++ /dev/null @@ -1,142 +0,0 @@ -from amarillo.tests.sampledata import carpool_1234, data1, carpool_repeating_json, stop_issue -from amarillo.services.gtfs_export import GtfsExport -from amarillo.services.gtfs import GtfsRtProducer -from amarillo.services.stops import StopsStore -from amarillo.services.trips import TripStore -from amarillo.models.Carpool import Carpool -from datetime import datetime -import time -import pytest - - -def test_gtfs_generation(): - cp = Carpool(**data1) - stops_store = StopsStore() - trips_store = TripStore(stops_store) - trips_store.put_carpool(cp) - - exporter = GtfsExport(None, None, trips_store, stops_store) - exporter.export('target/tests/test_gtfs_generation/test.gtfs.zip', "target/tests/test_gtfs_generation") - -def test_correct_stops(): - cp = Carpool(**stop_issue) - stops_store = StopsStore([{"url": "https://datahub.bbnavi.de/export/rideshare_points.geojson", "vicinity": 250}]) - stops_store.load_stop_sources() - trips_store = TripStore(stops_store) - trips_store.put_carpool(cp) - assert len(trips_store.trips) == 1 - - -class TestTripConverter: - - def setup_method(self, method): - self.stops_store = StopsStore([{"url": "https://datahub.bbnavi.de/export/rideshare_points.geojson", "vicinity": 50}]) - self.trips_store = TripStore(self.stops_store) - - def test_as_one_time_trip_as_delete_update(self): - cp = Carpool(**data1) - self.trips_store.put_carpool(cp) - trip = next(iter(self.trips_store.trips.values())) - - converter = GtfsRtProducer(self.trips_store) - json = converter._as_delete_updates(trip, datetime(2022,4,11)) - - assert json == [{ - 'trip': { - 'tripId': 'mfdz:Eins', - 'startTime': '23:59:00', - 'startDate': '20220530', - 'scheduleRelationship': 'CANCELED', - 'routeId': 'mfdz:Eins' - } - }] - - def test_as_one_time_trip_as_added_update(self): - cp = Carpool(**data1) - self.trips_store.put_carpool(cp) - trip = next(iter(self.trips_store.trips.values())) - - converter = GtfsRtProducer(self.trips_store) - json = converter._as_added_updates(trip, datetime(2022,4,11)) - assert json == [{ - 'trip': { - 'tripId': 'mfdz:Eins', - 'startTime': '23:59:00', - 'startDate': '20220530', - 'scheduleRelationship': 'ADDED', - 'routeId': 'mfdz:Eins', - '[transit_realtime.trip_descriptor]': { - 'routeUrl' : 'https://mfdz.de/trip/123', - 'agencyId' : 'mfdz', - 'route_long_name' : 'abc nach xyz', - 'route_type': 1551 - } - }, - 'stopTimeUpdate': [{ - 'stopSequence': 1, - 'arrival': { - 'time': time.mktime(datetime(2022,5,30,23,59,0).timetuple()), - 'uncertainty': 600 - }, - 'departure': { - 'time': time.mktime(datetime(2022,5,30,23,59,0).timetuple()), - 'uncertainty': 600 - }, - 'stopId': 'mfdz:12073:001', - 'scheduleRelationship': 'SCHEDULED', - 'stop_time_properties': { - '[transit_realtime.stop_time_properties]': { - 'dropoffType': 'NONE', - 'pickupType': 'COORDINATE_WITH_DRIVER' - } - } - }, - { - 'stopSequence': 2, - 'arrival': { - 'time': time.mktime(datetime(2022,5,31,0,16,45,0).timetuple()), - 'uncertainty': 600 - }, - 'departure': { - 'time': time.mktime(datetime(2022,5,31,0,16,45,0).timetuple()), - 'uncertainty': 600 - }, - - 'stopId': 'de:12073:900340137::3', - 'scheduleRelationship': 'SCHEDULED', - 'stop_time_properties': { - '[transit_realtime.stop_time_properties]': { - 'dropoffType': 'COORDINATE_WITH_DRIVER', - 'pickupType': 'NONE' - } - } - }] - }] - - def test_as_periodic_trip_as_delete_update(self): - cp = Carpool(**carpool_repeating_json) - self.trips_store.put_carpool(cp) - trip = next(iter(self.trips_store.trips.values())) - - converter = GtfsRtProducer(self.trips_store) - json = converter._as_delete_updates(trip, datetime(2022,4,11)) - - assert json == [{ - 'trip': { - 'tripId': 'mfdz:Zwei', - 'startTime': '15:00:00', - 'startDate': '20220411', - 'scheduleRelationship': 'CANCELED', - 'routeId': 'mfdz:Zwei' - } - }, - { - 'trip': { - 'tripId': 'mfdz:Zwei', - 'startTime': '15:00:00', - 'startDate': '20220418', - 'scheduleRelationship': 'CANCELED', - 'routeId': 'mfdz:Zwei' - } - } - ] \ No newline at end of file diff --git a/amarillo/tests/test_stops_gtfs_importer.py b/amarillo/tests/test_stops_gtfs_importer.py deleted file mode 100644 index 9777d18..0000000 --- a/amarillo/tests/test_stops_gtfs_importer.py +++ /dev/null @@ -1,15 +0,0 @@ -from amarillo.services.stop_importer.gtfs import GtfsStopsImporter - - -def test_load_stops_from_gtfs_(): - stopsDataFrames = GtfsStopsImporter().load_stops( - id='test_load_stops_from_gtfs', url='amarillo/tests/fixtures/stops.gtfs.zip' - ) - - assert len(stopsDataFrames) > 0 - assert stopsDataFrames.loc[0, ['x', 'y', 'stop_name', 'id']].values.tolist() == [ - 8.75033716398694, - 48.7891850492262, - 'Monakam Brunnenstr.', - 'de:08235:4060:0:3', - ] diff --git a/amarillo/tests/test_stops_overpass_importer.py b/amarillo/tests/test_stops_overpass_importer.py deleted file mode 100644 index 7e6a4b4..0000000 --- a/amarillo/tests/test_stops_overpass_importer.py +++ /dev/null @@ -1,14 +0,0 @@ -from amarillo.services.stop_importer.overpass import OverpassStopsImporter - - -def test_load_geojson_stops_from_web_(): - with open('amarillo/tests/fixtures/stops_overpass_result.csv') as f: - stopsDataFrames = OverpassStopsImporter()._parse_overpass_csv_response(f) - - assert len(stopsDataFrames) > 0 - assert stopsDataFrames.loc[0, ['x', 'y', 'stop_name', 'id']].values.tolist() == [ - 11.7621206, - 46.7470278, - 'P+R', - 'osm:n5206558994', - ] diff --git a/amarillo/tests/test_stops_store.py b/amarillo/tests/test_stops_store.py deleted file mode 100644 index 931a562..0000000 --- a/amarillo/tests/test_stops_store.py +++ /dev/null @@ -1,28 +0,0 @@ -from amarillo.models.Carpool import StopTime -from amarillo.services import stops - - -def test_load_stops_from_file(): - store = stops.StopsStore([{'url': 'amarillo/tests/stops.csv', 'vicinity': 50}]) - store.load_stop_sources() - assert len(store.stopsDataFrames[0]['stops']) > 0 - - -def test_load_csv_stops_from_web_(): - store = stops.StopsStore([{'url': 'https://data.mfdz.de/mfdz/stops/custom.csv', 'vicinity': 50}]) - store.load_stop_sources() - assert len(store.stopsDataFrames[0]['stops']) > 0 - - -def test_load_geojson_stops_from_web_(): - store = stops.StopsStore([{'url': 'https://datahub.bbnavi.de/export/rideshare_points.geojson', 'vicinity': 50}]) - store.load_stop_sources() - assert len(store.stopsDataFrames[0]['stops']) > 0 - - -def test_find_closest_stop(): - store = stops.StopsStore([{'url': 'amarillo/tests/stops.csv', 'vicinity': 50}]) - store.load_stop_sources() - carpool_stop = StopTime(name='start', lat=53.1191, lon=14.01577) - stop = store.find_closest_stop(carpool_stop, 1000) - assert stop.name == 'Mitfahrbank Biesenbrow' diff --git a/amarillo/tests/test_trip_store.py b/amarillo/tests/test_trip_store.py deleted file mode 100644 index 96c9616..0000000 --- a/amarillo/tests/test_trip_store.py +++ /dev/null @@ -1,23 +0,0 @@ -from amarillo.tests.sampledata import cp1, carpool_repeating -from amarillo.services.trips import TripStore -from amarillo.services.stops import StopsStore - - -import logging -logger = logging.getLogger(__name__) - -def test_trip_store_put_one_time_carpool(): - trip_store = TripStore(StopsStore()) - - t = trip_store.put_carpool(cp1) - assert t != None - assert len(t.stop_times) >= 2 - assert t.stop_times[0].stop_id == 'mfdz:12073:001' - assert t.stop_times[-1].stop_id == 'de:12073:900340137::3' - -def test_trip_store_put_repeating_carpool(): - trip_store = TripStore(StopsStore()) - - t = trip_store.put_carpool(carpool_repeating) - assert t != None - assert len(t.stop_times) >= 2 diff --git a/amarillo/utils/utils.py b/amarillo/utils/utils.py index c7c1075..93f7b61 100644 --- a/amarillo/utils/utils.py +++ b/amarillo/utils/utils.py @@ -1,8 +1,16 @@ import os import re +import shutil +from pathlib import Path +import logging + from datetime import datetime, date, timedelta from pyproj import Geod +logger = logging.getLogger(__name__) +#logging.conf may not exist yet, so we need to configure the logger to show infos +logging.basicConfig(level=logging.INFO) + def assert_folder_exists(foldername): if not os.path.isdir(foldername): os.makedirs(foldername, exist_ok=True) @@ -37,4 +45,36 @@ def geodesic_distance_in_m(coord1, coord2): geod = Geod(ellps="WGS84") lons = [coord1[0], coord2[0]] lats = [coord1[1], coord2[1]] - return geod.line_lengths(lons, lats)[0] \ No newline at end of file + return geod.line_lengths(lons, lats)[0] + +def carpool_distance_in_m(carpool): + if len(carpool.stops) < 2: + return 0 + s1 = carpool.stops[0] + s2 = carpool.stops[-1] + return geodesic_distance_in_m((s1.lon, s1.lat),(s2.lon, s2.lat)) + +def copy_static_files(files_and_dirs_to_copy): + amarillo_dir = Path(__file__).parents[1] + source_dir = os.path.join(amarillo_dir, "static") + + destination_dir = os.getcwd() + + for item in files_and_dirs_to_copy: + source_path = os.path.join(source_dir, item) + destination_path = os.path.join(destination_dir, item) + + if not os.path.exists(source_path): + raise FileNotFoundError(source_path) + + if os.path.exists(destination_path): + # logger.info(f"{item} already exists") + continue + + if os.path.isfile(source_path): + shutil.copy2(source_path, destination_path) + logger.info(f"Copied {item} to {destination_path}") + + if os.path.isdir(source_path): + shutil.copytree(source_path, destination_path) + logger.info(f"Copied directory {item} and its contents to {destination_path}") \ No newline at end of file diff --git a/enhancer.py b/enhancer.py deleted file mode 100644 index 1bdbb87..0000000 --- a/enhancer.py +++ /dev/null @@ -1,70 +0,0 @@ -import json -import time -import logging -import logging.config -from watchdog.observers import Observer -from watchdog.events import FileSystemEventHandler - -from amarillo.configuration import configure_enhancer_services -from amarillo.utils.container import container -from amarillo.models.Carpool import Carpool -from amarillo.utils.utils import agency_carpool_ids_from_filename - -logging.config.fileConfig('logging.conf', disable_existing_loggers=False) -logger = logging.getLogger("enhancer") - -logger.info("Hello Enhancer") - -configure_enhancer_services() - -observer = Observer() # Watch Manager - - -class EventHandler(FileSystemEventHandler): - # TODO FG HB should watch for both carpools and agencies - # in data/agency, data/agencyconf, see AgencyConfService - - def on_closed(self, event): - - logger.info("CLOSE_WRITE: Created %s", event.src_path) - try: - with open(event.src_path, 'r', encoding='utf-8') as f: - dict = json.load(f) - carpool = Carpool(**dict) - - container['carpools'].put(carpool.agency, carpool.id, carpool) - except FileNotFoundError as e: - logger.error("Carpool could not be added, as already deleted (%s)", event.src_path) - except: - logger.exception("Eventhandler on_closed encountered exception") - - def on_deleted(self, event): - try: - logger.info("DELETE: Removing %s", event.src_path) - (agency_id, carpool_id) = agency_carpool_ids_from_filename(event.src_path) - container['carpools'].delete(agency_id, carpool_id) - except: - logger.exception("Eventhandler on_deleted encountered exception") - - -observer.schedule(EventHandler(), 'data/carpool', recursive=True) -observer.start() - -import time - -try: - # TODO FG Is this really needed? - cnt = 0 - ENHANCER_LOG_INTERVAL_IN_S = 600 - while True: - if cnt == ENHANCER_LOG_INTERVAL_IN_S: - logger.debug("Currently stored carpool ids: %s", container['carpools'].get_all_ids()) - cnt = 0 - - time.sleep(1) - cnt += 1 -finally: - observer.stop() - observer.join() - - logger.info("Goodbye Enhancer") diff --git a/logging.conf b/logging.conf deleted file mode 100644 index 429da8e..0000000 --- a/logging.conf +++ /dev/null @@ -1,22 +0,0 @@ -[loggers] -keys=root - -[handlers] -keys=consoleHandler - -[formatters] -keys=simpleFormatter - -[logger_root] -level=INFO -handlers=consoleHandler -propagate=yes - -[handler_consoleHandler] -class=StreamHandler -level=DEBUG -formatter=simpleFormatter -args=(sys.stdout,) - -[formatter_simpleFormatter] -format=%(asctime)s - %(name)s - %(levelname)s - %(message)s \ No newline at end of file diff --git a/mitanand.Dockerfile b/mitanand.Dockerfile new file mode 100644 index 0000000..de7543a --- /dev/null +++ b/mitanand.Dockerfile @@ -0,0 +1,22 @@ +ARG DOCKER_REGISTRY +FROM ${DOCKER_REGISTRY}/amarillo/amarillo-base + +ARG PLUGINS=\ +"amarillo-metrics "\ +"amarillo-grfs-exporter " + +ARG PACKAGE_REGISTRY_URL + +ENV METRICS_USER='' +ENV METRICS_PASSWORD='' + +# RUN pip install $PLUGINS + +RUN pip install --no-cache-dir --upgrade --extra-index-url ${PACKAGE_REGISTRY_URL} ${PLUGINS} + +# Create the error.log, otherwise we get a permission error when we try to write to it +RUN touch /app/error.log +RUN chmod 777 /app/error.log + +RUN useradd amarillo +USER amarillo \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index d9f1f5d..9fa511a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,27 @@ +[project] +name = "amarillo" +version = "2.0.1" +description = "Aggregates and enhances carpooling-offers and publishes them as GTFS(-RT)" +readme = "README.md" +license = {file = "LICENSE"} +keywords = ["amarillo", "ridesharing", "carpooling", "gtfs", "gtfs-rt"] +dependencies = [ + "fastapi[all]==0.111.0", + "uvicorn[standard]~=0.29.0", + "pydantic[dotenv]==2.4.2", + "rtree==1.1.0", + "setproctitle==1.3.3", + "starlette~=0.35", + "requests==2.31.0", + "pyproj==3.6.1", + "geojson-pydantic==1.0.1", + "idna>=3.7", + "numpy==1.26.4", +] + +[tool.setuptools.packages] +find = {} + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" @@ -72,3 +96,4 @@ ignore_missing_imports = true [[tool.mypy.overrides]] # See https://github.com/HBNetwork/python-decouple/issues/122 module = ["geopandas"] + diff --git a/requirements.txt b/requirements.txt index 006e29e..4314b77 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,11 @@ -fastapi[all]==0.109.0 -geopandas==0.14 -uvicorn[standard]==0.23.2 +fastapi[all]==0.111.0 +uvicorn[standard]~=0.29.0 pydantic[dotenv]==2.4.2 -protobuf==3.20.3 rtree==1.1.0 -schedule==1.2.1 setproctitle==1.3.3 starlette~=0.35 -pandas==2.1.1 requests==2.31.0 -Shapely==2.0.2 -pygeos==0.14 pyproj==3.6.1 geojson-pydantic==1.0.1 -watchdog==3.0.0 +idna>=3.7 +numpy==1.26.4 \ No newline at end of file diff --git a/standard.Dockerfile b/standard.Dockerfile new file mode 100644 index 0000000..d942f74 --- /dev/null +++ b/standard.Dockerfile @@ -0,0 +1,22 @@ +ARG DOCKER_REGISTRY +FROM ${DOCKER_REGISTRY}/amarillo/amarillo-base + +ARG PLUGINS=\ +"amarillo-metrics "\ +"amarillo-gtfs-exporter " + +ARG PACKAGE_REGISTRY_URL + +ENV METRICS_USER='' +ENV METRICS_PASSWORD='' + +# RUN pip install $PLUGINS + +RUN pip install --no-cache-dir --upgrade --extra-index-url ${PACKAGE_REGISTRY_URL} ${PLUGINS} + +# Create the error.log, otherwise we get a permission error when we try to write to it +RUN touch /app/error.log +RUN chmod 777 /app/error.log + +RUN useradd amarillo +USER amarillo \ No newline at end of file