From ad0689e7242b815d6f281b78f12962abed7220b2 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 25 Jan 2024 15:31:58 +0100 Subject: [PATCH 01/50] added pyproject.toml --- pyproject.toml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index d9f1f5d..af5be75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,26 @@ +[project] +name = "amarillo" +version = "0.0.15a3" +description = "Aggregates and enhances carpooling-offers and publishes them as GTFS(-RT)" +readme = "README.md" +license = {file = "LICENSE"} +keywords = ["amarillo", "ridesharing", "carpooling", "gtfs", "gtfs-rt"] +dependencies = [ + "fastapi[all]==0.109.0", + "geopandas==0.14", + "uvicorn[standard]==0.23.2", + "pydantic[dotenv]==2.4.2", + "protobuf==3.20.3", + "starlette~=0.35", + "requests==2.31.0", + "pyproj==3.6.1", + "geojson-pydantic==1.0.1", + "watchdog==3.0.0", +] + +[tool.setuptools.packages] +find = {} + [build-system] requires = ["setuptools"] build-backend = "setuptools.build_meta" @@ -72,3 +95,4 @@ ignore_missing_imports = true [[tool.mypy.overrides]] # See https://github.com/HBNetwork/python-decouple/issues/122 module = ["geopandas"] + From 7ac3ffe727280b6ba771b2bdcd4305de97207fc9 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 25 Jan 2024 15:32:21 +0100 Subject: [PATCH 02/50] plugin discovery --- amarillo/main.py | 35 ++++++++++++++++++++++++++++------- amarillo/plugins/__init__.py | 1 + 2 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 amarillo/plugins/__init__.py diff --git a/amarillo/main.py b/amarillo/main.py index fc1ef12..cadafe3 100644 --- a/amarillo/main.py +++ b/amarillo/main.py @@ -1,21 +1,22 @@ import logging.config - -from amarillo.configuration import configure_services, configure_admin_token -from amarillo.services.config import config - -logging.config.fileConfig('logging.conf', disable_existing_loggers=False) -logger = logging.getLogger("main") - +import importlib +import pkgutil import uvicorn import mimetypes from starlette.staticfiles import StaticFiles +import amarillo.plugins +from amarillo.services.config import config +from amarillo.configuration import configure_services, configure_admin_token from amarillo.routers import carpool, agency, agencyconf, region from fastapi import FastAPI # https://pydantic-docs.helpmanual.io/usage/settings/ from amarillo.views import home +logging.config.fileConfig('logging.conf', disable_existing_loggers=False) +logger = logging.getLogger("main") + logger.info("Hello Amarillo!") app = FastAPI(title="Amarillo - The Carpooling Intermediary", @@ -79,10 +80,30 @@ app.include_router(region.router) +def iter_namespace(ns_pkg): + # Source: https://packaging.python.org/guides/creating-and-discovering-plugins/ + return pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".") + +def load_plugins(): + discovered_plugins = { + name: importlib.import_module(name) + for finder, name, ispkg + in iter_namespace(amarillo.plugins) + } + logger.info(f"Discovered plugins: {list(discovered_plugins.keys())}") + + for name, module in discovered_plugins.items(): + if hasattr(module, "setup"): + logger.info(f"Running setup function for {name}") + module.setup(app) + + else: logger.info(f"Did not find setup function for {name}") + def configure(): configure_admin_token() configure_services() configure_routing() + load_plugins() def configure_routing(): diff --git a/amarillo/plugins/__init__.py b/amarillo/plugins/__init__.py new file mode 100644 index 0000000..0260537 --- /dev/null +++ b/amarillo/plugins/__init__.py @@ -0,0 +1 @@ +__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file From 57c3f74903042c76b97cf71161af42321b558858 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 25 Jan 2024 15:33:03 +0100 Subject: [PATCH 03/50] metrics secrets --- amarillo/services/secrets.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/amarillo/services/secrets.py b/amarillo/services/secrets.py index 1afd93d..5085fce 100644 --- a/amarillo/services/secrets.py +++ b/amarillo/services/secrets.py @@ -1,10 +1,13 @@ from typing import Dict from pydantic import Field from pydantic_settings import BaseSettings - +from typing import Optional # Example: secrets = { "mfdz": "some secret" } class Secrets(BaseSettings): ride2go_token: str = Field(None, env = 'RIDE2GO_TOKEN') + # TODO: define these as required if metrics plugin is installed + metrics_user: Optional[str] = Field(None, env = 'METRICS_USER') + metrics_password: Optional[str] = Field(None, env = 'METRICS_PASSWORD') # Read if file exists, otherwise no error (it's in .gitignore) From 8129e826aa38d186cd4490c2d778dae26fc42913 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 1 Feb 2024 15:51:37 +0100 Subject: [PATCH 04/50] install plugins in Dockerfile --- Dockerfile | 4 +++- README.md | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9575a5c..6002b0a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,8 +20,10 @@ ENV RIDE2GO_TOKEN='' EXPOSE 80 +ARG PLUGINS + COPY requirements.txt /app/requirements.txt -RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt +RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt ${PLUGINS} # set MODULE_NAME explicitly ENV MODULE_NAME=amarillo.main diff --git a/README.md b/README.md index d78bad7..329912b 100644 --- a/README.md +++ b/README.md @@ -62,5 +62,5 @@ In the top directory, run `pytest amarillo/tests`. Based on [tiangolo/uvicorn-gunicorn:python3.9-slim](https://github.com/tiangolo/uvicorn-gunicorn-docker) -- build `docker build -t amarillo .` +- build `docker build -t amarillo --build-arg="PLUGINS=amarillo-metrics" .` - run `docker run --rm --name amarillo -p 8000:80 -e MAX_WORKERS="1" -e ADMIN_TOKEN=$ADMIN_TOKEN -e RIDE2GO_TOKEN=$RIDE2GO_TOKEN -e TZ=Europe/Berlin -v $(pwd)/data:/app/data amarillo` From 338ecd7200e019604a8aa23c3e9f9932d103a993 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 29 Feb 2024 15:14:43 +0100 Subject: [PATCH 05/50] Moved metric user and password configuration to metrics plugin --- amarillo/services/secrets.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/amarillo/services/secrets.py b/amarillo/services/secrets.py index 5085fce..756d21c 100644 --- a/amarillo/services/secrets.py +++ b/amarillo/services/secrets.py @@ -1,13 +1,9 @@ -from typing import Dict -from pydantic import Field +from pydantic import Field, ConfigDict from pydantic_settings import BaseSettings -from typing import Optional # Example: secrets = { "mfdz": "some secret" } class Secrets(BaseSettings): + model_config = ConfigDict(extra='allow') ride2go_token: str = Field(None, env = 'RIDE2GO_TOKEN') - # TODO: define these as required if metrics plugin is installed - metrics_user: Optional[str] = Field(None, env = 'METRICS_USER') - metrics_password: Optional[str] = Field(None, env = 'METRICS_PASSWORD') # Read if file exists, otherwise no error (it's in .gitignore) From 7a5a89ffb5f1e90b7cca35b3e29d8f35869b4c90 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Fri, 22 Mar 2024 16:21:09 +0100 Subject: [PATCH 06/50] Added amarillo-metrics to requirements.txt --- README.md | 2 +- requirements.txt | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 329912b..15f1fe9 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ In the top directory, run `pytest amarillo/tests`. ## Docker -Based on [tiangolo/uvicorn-gunicorn:python3.9-slim](https://github.com/tiangolo/uvicorn-gunicorn-docker) +Based on [tiangolo/uvicorn-gunicorn:python3.10-slim](https://github.com/tiangolo/uvicorn-gunicorn-docker) - build `docker build -t amarillo --build-arg="PLUGINS=amarillo-metrics" .` - run `docker run --rm --name amarillo -p 8000:80 -e MAX_WORKERS="1" -e ADMIN_TOKEN=$ADMIN_TOKEN -e RIDE2GO_TOKEN=$RIDE2GO_TOKEN -e TZ=Europe/Berlin -v $(pwd)/data:/app/data amarillo` diff --git a/requirements.txt b/requirements.txt index 006e29e..292a8f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +amarillo-metrics==0.0.6 fastapi[all]==0.109.0 geopandas==0.14 uvicorn[standard]==0.23.2 @@ -13,4 +14,4 @@ Shapely==2.0.2 pygeos==0.14 pyproj==3.6.1 geojson-pydantic==1.0.1 -watchdog==3.0.0 +watchdog==3.0.0 \ No newline at end of file From d539b04925f6e8bd9591d78a448ca4fb391dd65b Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 6 Feb 2024 12:05:27 +0100 Subject: [PATCH 07/50] moved static files to amarillo/static --- .gitignore | 7 +++++++ Dockerfile | 11 ++++++----- {conf => amarillo/static/conf}/agency/mfdz.json | 0 {conf => amarillo/static/conf}/agency/mifaz.json | 0 {conf => amarillo/static/conf}/agency/ride2go.json | 0 {conf => amarillo/static/conf}/agency/ummadum.json | 0 {conf => amarillo/static/conf}/region/bb.json | 0 {conf => amarillo/static/conf}/region/bw.json | 0 {conf => amarillo/static/conf}/region/by.json | 0 {conf => amarillo/static/conf}/region/nrw.json | 0 {conf => amarillo/static/conf}/stop_sources.json | 0 config => amarillo/static/config | 0 logging.conf => amarillo/static/logging.conf | 0 {static => amarillo/static/static}/css/docs.css | 0 {static => amarillo/static/static}/css/theme.css | 0 {static => amarillo/static/static}/img/cloud.png | Bin {static => amarillo/static/static}/img/favicon.ico | Bin .../static/templates}/home/index.html | 0 .../static/templates}/shared/layout.html | 0 19 files changed, 13 insertions(+), 5 deletions(-) rename {conf => amarillo/static/conf}/agency/mfdz.json (100%) rename {conf => amarillo/static/conf}/agency/mifaz.json (100%) rename {conf => amarillo/static/conf}/agency/ride2go.json (100%) rename {conf => amarillo/static/conf}/agency/ummadum.json (100%) rename {conf => amarillo/static/conf}/region/bb.json (100%) rename {conf => amarillo/static/conf}/region/bw.json (100%) rename {conf => amarillo/static/conf}/region/by.json (100%) rename {conf => amarillo/static/conf}/region/nrw.json (100%) rename {conf => amarillo/static/conf}/stop_sources.json (100%) rename config => amarillo/static/config (100%) rename logging.conf => amarillo/static/logging.conf (100%) rename {static => amarillo/static/static}/css/docs.css (100%) rename {static => amarillo/static/static}/css/theme.css (100%) rename {static => amarillo/static/static}/img/cloud.png (100%) rename {static => amarillo/static/static}/img/favicon.ico (100%) rename {templates => amarillo/static/templates}/home/index.html (100%) rename {templates => amarillo/static/templates}/shared/layout.html (100%) diff --git a/.gitignore b/.gitignore index 83461d6..6d0e3dc 100644 --- a/.gitignore +++ b/.gitignore @@ -132,6 +132,7 @@ setup.ipynb *~ /.idea/ +/.vscode/ secrets gtfs/*.zip gtfs/*.pbf @@ -143,3 +144,9 @@ data/trash/ data/gtfs/ data/tmp +#these files are under app/static but they get copied to the outside directory on startup +logging.conf +config +static/** +templates/** +conf/** diff --git a/Dockerfile b/Dockerfile index 6002b0a..d1babed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,11 +31,12 @@ ENV MODULE_NAME=amarillo.main COPY ./amarillo /app/amarillo COPY enhancer.py /app COPY prestart.sh /app -COPY ./static /app/static -COPY ./templates /app/templates -COPY config /app -COPY logging.conf /app -COPY ./conf /app/conf +COPY ./amarillo/plugins /app/amarillo/plugins +COPY ./amarillo/static/static /app/static +COPY ./amarillo/static/templates /app/templates +COPY ./amarillo/static/config /app +COPY ./amarillo/static/logging.conf /app +COPY ./amarillo/static/conf /app/conf # This image inherits uvicorn-gunicorn's CMD. If you'd like to start uvicorn, use this instead # CMD ["uvicorn", "amarillo.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/conf/agency/mfdz.json b/amarillo/static/conf/agency/mfdz.json similarity index 100% rename from conf/agency/mfdz.json rename to amarillo/static/conf/agency/mfdz.json diff --git a/conf/agency/mifaz.json b/amarillo/static/conf/agency/mifaz.json similarity index 100% rename from conf/agency/mifaz.json rename to amarillo/static/conf/agency/mifaz.json diff --git a/conf/agency/ride2go.json b/amarillo/static/conf/agency/ride2go.json similarity index 100% rename from conf/agency/ride2go.json rename to amarillo/static/conf/agency/ride2go.json diff --git a/conf/agency/ummadum.json b/amarillo/static/conf/agency/ummadum.json similarity index 100% rename from conf/agency/ummadum.json rename to amarillo/static/conf/agency/ummadum.json diff --git a/conf/region/bb.json b/amarillo/static/conf/region/bb.json similarity index 100% rename from conf/region/bb.json rename to amarillo/static/conf/region/bb.json diff --git a/conf/region/bw.json b/amarillo/static/conf/region/bw.json similarity index 100% rename from conf/region/bw.json rename to amarillo/static/conf/region/bw.json diff --git a/conf/region/by.json b/amarillo/static/conf/region/by.json similarity index 100% rename from conf/region/by.json rename to amarillo/static/conf/region/by.json diff --git a/conf/region/nrw.json b/amarillo/static/conf/region/nrw.json similarity index 100% rename from conf/region/nrw.json rename to amarillo/static/conf/region/nrw.json diff --git a/conf/stop_sources.json b/amarillo/static/conf/stop_sources.json similarity index 100% rename from conf/stop_sources.json rename to amarillo/static/conf/stop_sources.json diff --git a/config b/amarillo/static/config similarity index 100% rename from config rename to amarillo/static/config diff --git a/logging.conf b/amarillo/static/logging.conf similarity index 100% rename from logging.conf rename to amarillo/static/logging.conf diff --git a/static/css/docs.css b/amarillo/static/static/css/docs.css similarity index 100% rename from static/css/docs.css rename to amarillo/static/static/css/docs.css diff --git a/static/css/theme.css b/amarillo/static/static/css/theme.css similarity index 100% rename from static/css/theme.css rename to amarillo/static/static/css/theme.css diff --git a/static/img/cloud.png b/amarillo/static/static/img/cloud.png similarity index 100% rename from static/img/cloud.png rename to amarillo/static/static/img/cloud.png diff --git a/static/img/favicon.ico b/amarillo/static/static/img/favicon.ico similarity index 100% rename from static/img/favicon.ico rename to amarillo/static/static/img/favicon.ico diff --git a/templates/home/index.html b/amarillo/static/templates/home/index.html similarity index 100% rename from templates/home/index.html rename to amarillo/static/templates/home/index.html diff --git a/templates/shared/layout.html b/amarillo/static/templates/shared/layout.html similarity index 100% rename from templates/shared/layout.html rename to amarillo/static/templates/shared/layout.html From e1a35fa7830f3f85f0ec9a427a9ff69648d8d52a Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 6 Feb 2024 12:58:36 +0100 Subject: [PATCH 08/50] copy static files to working directory --- MANIFEST.in | 2 ++ amarillo/main.py | 4 ++++ amarillo/utils/utils.py | 36 +++++++++++++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 MANIFEST.in diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..0daae06 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include amarillo/static/ * +recursive-include amarillo/tests/ * \ No newline at end of file diff --git a/amarillo/main.py b/amarillo/main.py index cadafe3..19b4ec5 100644 --- a/amarillo/main.py +++ b/amarillo/main.py @@ -5,6 +5,10 @@ import mimetypes from starlette.staticfiles import StaticFiles +from amarillo.utils.utils import copy_static_files +#this has to run before app.configuration is imported, otherwise we get validation error for config because the config file is not copied yet +copy_static_files(["conf", "static", "templates", "logging.conf", "config"]) + import amarillo.plugins from amarillo.services.config import config from amarillo.configuration import configure_services, configure_admin_token diff --git a/amarillo/utils/utils.py b/amarillo/utils/utils.py index c7c1075..9d060b3 100644 --- a/amarillo/utils/utils.py +++ b/amarillo/utils/utils.py @@ -1,8 +1,16 @@ import os import re +import shutil +from pathlib import Path +import logging + from datetime import datetime, date, timedelta from pyproj import Geod +logger = logging.getLogger(__name__) +#logging.conf may not exist yet, so we need to configure the logger to show infos +logging.basicConfig(level=logging.INFO) + def assert_folder_exists(foldername): if not os.path.isdir(foldername): os.makedirs(foldername, exist_ok=True) @@ -37,4 +45,30 @@ def geodesic_distance_in_m(coord1, coord2): geod = Geod(ellps="WGS84") lons = [coord1[0], coord2[0]] lats = [coord1[1], coord2[1]] - return geod.line_lengths(lons, lats)[0] \ No newline at end of file + return geod.line_lengths(lons, lats)[0] + + +def copy_static_files(files_and_dirs_to_copy): + amarillo_dir = Path(__file__).parents[1] + source_dir = os.path.join(amarillo_dir, "static") + + destination_dir = os.getcwd() + + for item in files_and_dirs_to_copy: + source_path = os.path.join(source_dir, item) + destination_path = os.path.join(destination_dir, item) + + if not os.path.exists(source_path): + raise FileNotFoundError(source_path) + + if os.path.exists(destination_path): + # logger.info(f"{item} already exists") + continue + + if os.path.isfile(source_path): + shutil.copy2(source_path, destination_path) + logger.info(f"Copied {item} to {destination_path}") + + if os.path.isdir(source_path): + shutil.copytree(source_path, destination_path) + logger.info(f"Copied directory {item} and its contents to {destination_path}") \ No newline at end of file From 421d7c0fb2400c3d9d12834d9049a255fddd1cf1 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 30 Jan 2024 15:08:38 +0100 Subject: [PATCH 09/50] moved gtfs generation and enhancer.py to enhancer plugin --- amarillo/services/gtfs_export.py | 229 ---------------------------- amarillo/services/gtfs_generator.py | 71 --------- enhancer.py | 70 --------- 3 files changed, 370 deletions(-) delete mode 100644 amarillo/services/gtfs_export.py delete mode 100644 amarillo/services/gtfs_generator.py delete mode 100644 enhancer.py diff --git a/amarillo/services/gtfs_export.py b/amarillo/services/gtfs_export.py deleted file mode 100644 index fb39425..0000000 --- a/amarillo/services/gtfs_export.py +++ /dev/null @@ -1,229 +0,0 @@ - -from collections.abc import Iterable -from datetime import datetime, timedelta -from zipfile import ZipFile -import csv -import gettext -import logging -import re - -from amarillo.utils.utils import assert_folder_exists -from amarillo.models.gtfs import GtfsTimeDelta, GtfsFeedInfo, GtfsAgency, GtfsRoute, GtfsStop, GtfsStopTime, GtfsTrip, GtfsCalendar, GtfsCalendarDate, GtfsShape -from amarillo.services.stops import is_carpooling_stop -from amarillo.services.gtfs_constants import * - - -logger = logging.getLogger(__name__) - -class GtfsExport: - - stops_counter = 0 - trips_counter = 0 - routes_counter = 0 - - stored_stops = {} - - def __init__(self, agencies, feed_info, ridestore, stopstore, bbox = None): - self.stops = {} - self.routes = [] - self.calendar_dates = [] - self.calendar = [] - self.trips = [] - self.stop_times = [] - self.calendar = [] - self.shapes = [] - self.agencies = agencies - self.feed_info = feed_info - self.localized_to = " nach " - self.localized_short_name = "Mitfahrgelegenheit" - self.stopstore = stopstore - self.ridestore = ridestore - self.bbox = bbox - - def export(self, gtfszip_filename, gtfsfolder): - assert_folder_exists(gtfsfolder) - self._prepare_gtfs_feed(self.ridestore, self.stopstore) - self._write_csvfile(gtfsfolder, 'agency.txt', self.agencies) - self._write_csvfile(gtfsfolder, 'feed_info.txt', self.feed_info) - self._write_csvfile(gtfsfolder, 'routes.txt', self.routes) - self._write_csvfile(gtfsfolder, 'trips.txt', self.trips) - self._write_csvfile(gtfsfolder, 'calendar.txt', self.calendar) - self._write_csvfile(gtfsfolder, 'calendar_dates.txt', self.calendar_dates) - self._write_csvfile(gtfsfolder, 'stops.txt', self.stops.values()) - self._write_csvfile(gtfsfolder, 'stop_times.txt', self.stop_times) - self._write_csvfile(gtfsfolder, 'shapes.txt', self.shapes) - self._zip_files(gtfszip_filename, gtfsfolder) - - def _zip_files(self, gtfszip_filename, gtfsfolder): - gtfsfiles = ['agency.txt', 'feed_info.txt', 'routes.txt', 'trips.txt', - 'calendar.txt', 'calendar_dates.txt', 'stops.txt', 'stop_times.txt', 'shapes.txt'] - with ZipFile(gtfszip_filename, 'w') as gtfszip: - for gtfsfile in gtfsfiles: - gtfszip.write(gtfsfolder+'/'+gtfsfile, gtfsfile) - - def _prepare_gtfs_feed(self, ridestore, stopstore): - """ - Prepares all gtfs objects in memory before they are written - to their respective streams. - - For all wellknown stops a GTFS stop is created and - afterwards all ride offers are transformed into their - gtfs equivalents. - """ - for stopSet in stopstore.stopsDataFrames: - for stop in stopSet["stops"].itertuples(): - self._load_stored_stop(stop) - cloned_trips = dict(ridestore.trips) - for url, trip in cloned_trips.items(): - if self.bbox is None or trip.intersects(self.bbox): - self._convert_trip(trip) - - def _convert_trip(self, trip): - self.routes_counter += 1 - self.routes.append(self._create_route(trip)) - self.calendar.append(self._create_calendar(trip)) - if not trip.runs_regularly: - self.calendar_dates.append(self._create_calendar_date(trip)) - self.trips.append(self._create_trip(trip, self.routes_counter)) - self._append_stops_and_stop_times(trip) - self._append_shapes(trip, self.routes_counter) - - def _trip_headsign(self, destination): - destination = destination.replace('(Deutschland)', '') - destination = destination.replace(', Deutschland', '') - appendix = '' - if 'Schweiz' in destination or 'Switzerland' in destination: - appendix = ', Schweiz' - destination = destination.replace('(Schweiz)', '') - destination = destination.replace(', Schweiz', '') - destination = destination.replace('(Switzerland)', '') - - try: - matches = re.match(r"(.*,)? ?(\d{4,5})? ?(.*)", destination) - - match = matches.group(3).strip() if matches != None else destination.strip() - if match[-1]==')' and not '(' in match: - match = match[0:-1] - - return match + appendix - except Exception as ex: - logger.error("error for "+destination ) - logger.exception(ex) - return destination - - def _create_route(self, trip): - return GtfsRoute(trip.agency, trip.trip_id, trip.route_long_name(), RIDESHARING_ROUTE_TYPE, trip.url, "") - - def _create_calendar(self, trip): - # TODO currently, calendar is not provided by Fahrgemeinschaft.de interface. - # We could apply some heuristics like requesting multiple days and extrapolate - # if multiple trips are found, but better would be to have these provided by the - # offical interface. Then validity periods should be provided as well (not - # sure if these are available) - # For fahrgemeinschaft.de, regurlar trips are recognizable via their url - # which contains "regelmaessig". However, we don't know on which days of the week, - # nor until when. As a first guess, if datetime is a mo-fr, we assume each workday, - # if it's sa/su, only this... - - feed_start_date = datetime.today() - stop_date = self._convert_stop_date(feed_start_date) - return GtfsCalendar(trip.trip_id, stop_date, self._convert_stop_date(feed_start_date + timedelta(days=31)), *(trip.weekdays)) - - def _create_calendar_date(self, trip): - return GtfsCalendarDate(trip.trip_id, self._convert_stop_date(trip.start), CALENDAR_DATES_EXCEPTION_TYPE_ADDED) - - def _create_trip(self, trip, shape_id): - return GtfsTrip(trip.trip_id, trip.trip_id, trip.trip_id, shape_id, trip.trip_headsign, NO_BIKES_ALLOWED) - - def _convert_stop(self, stop): - """ - Converts a stop represented as pandas row to a gtfs stop. - Expected attributes of stop: id, stop_name, x, y (in wgs84) - """ - if stop.id: - id = stop.id - else: - self.stops_counter += 1 - id = "tmp-{}".format(self.stops_counter) - - stop_name = "k.A." if stop.stop_name is None else stop.stop_name - return GtfsStop(id, stop.y, stop.x, stop_name) - - def _append_stops_and_stop_times(self, trip): - # Assumptions: - # arrival_time = departure_time - # pickup_type, drop_off_type for origin: = coordinate/none - # pickup_type, drop_off_type for destination: = none/coordinate - # timepoint = approximate for origin and destination (not sure what consequences this might have for trip planners) - for stop_time in trip.stop_times: - # retrieve stop from stored_stops and mark it to be exported - wkn_stop = self.stored_stops.get(stop_time.stop_id) - if not wkn_stop: - logger.warning("No stop found in stop_store for %s. Will skip stop_time %s of trip %s", stop_time.stop_id, stop_time.stop_sequence, trip.trip_id) - else: - self.stops[stop_time.stop_id] = wkn_stop - # Append stop_time - self.stop_times.append(stop_time) - - def _append_shapes(self, trip, shape_id): - counter = 0 - for point in trip.path.coordinates: - counter += 1 - self.shapes.append(GtfsShape(shape_id, point[0], point[1], counter)) - - def _stop_hash(self, stop): - return "{}#{}#{}".format(stop.stop_name,stop.x,stop.y) - - def _should_always_export(self, stop): - """ - Returns true, if the given stop shall be exported to GTFS, - regardless, if it's part of a trip or not. - - This is necessary, as potential stops are required - to be part of the GTFS to be referenced later on - by dynamicly added trips. - """ - if self.bbox: - return (self.bbox[0] <= stop.stop_lon <= self.bbox[2] and - self.bbox[1] <= stop.stop_lat <= self.bbox[3]) - else: - return is_carpooling_stop(stop.stop_id, stop.stop_name) - - def _load_stored_stop(self, stop): - gtfsstop = self._convert_stop(stop) - stop_hash = self._stop_hash(stop) - self.stored_stops[gtfsstop.stop_id] = gtfsstop - if self._should_always_export(gtfsstop): - self.stops[gtfsstop.stop_id] = gtfsstop - - def _get_stop_by_hash(self, stop_hash): - return self.stops.get(stop_hash, self.stored_stops.get(stop_hash)) - - def _get_or_create_stop(self, stop): - stop_hash = self._stop_hash(stop) - gtfsstop = self.stops.get(stop_hash) - if gtfsstop is None: - gtfsstop = self.stored_stops.get(stop_hash, self._convert_stop(stop)) - self.stops[stop_hash] = gtfsstop - return gtfsstop - - def _convert_stop_date(self, date_time): - return date_time.strftime("%Y%m%d") - - def _write_csvfile(self, gtfsfolder, filename, content): - with open(gtfsfolder+"/"+filename, 'w', newline="\n", encoding="utf-8") as csvfile: - self._write_csv(csvfile, content) - - def _write_csv(self, csvfile, content): - if hasattr(content, '_fields'): - writer = csv.DictWriter(csvfile, content._fields) - writer.writeheader() - writer.writerow(content._asdict()) - else: - if content: - writer = csv.DictWriter(csvfile, next(iter(content))._fields) - writer.writeheader() - for record in content: - writer.writerow(record._asdict()) - - \ No newline at end of file diff --git a/amarillo/services/gtfs_generator.py b/amarillo/services/gtfs_generator.py deleted file mode 100644 index 113d5db..0000000 --- a/amarillo/services/gtfs_generator.py +++ /dev/null @@ -1,71 +0,0 @@ -from amarillo.models.Carpool import Region -from amarillo.services.gtfs_export import GtfsExport, GtfsFeedInfo, GtfsAgency -from amarillo.services.gtfs import GtfsRtProducer -from amarillo.utils.container import container -from glob import glob -import json -import schedule -import threading -import time -import logging -from datetime import date, timedelta - -logger = logging.getLogger(__name__) - -regions = {} -for region_file_name in glob('conf/region/*.json'): - with open(region_file_name) as region_file: - dict = json.load(region_file) - region = Region(**dict) - region_id = region.id - regions[region_id] = region - -agencies = [] -for agency_file_name in glob('conf/agency/*.json'): - with open(agency_file_name) as agency_file: - dict = json.load(agency_file) - agency = GtfsAgency(dict["id"], dict["name"], dict["url"], dict["timezone"], dict["lang"], dict["email"]) - agency_id = agency.agency_id - agencies.append(agency) - -def run_schedule(): - while 1: - try: - schedule.run_pending() - except Exception as e: - logger.exception(e) - time.sleep(1) - -def midnight(): - container['stops_store'].load_stop_sources() - container['trips_store'].unflag_unrecent_updates() - container['carpools'].purge_outdated_offers() - generate_gtfs() - -def generate_gtfs(): - logger.info("Generate GTFS") - - for region in regions.values(): - # TODO make feed producer infos configurable - feed_info = GtfsFeedInfo('mfdz', 'MITFAHR|DE|ZENTRALE', 'http://www.mitfahrdezentrale.de', 'de', 1) - exporter = GtfsExport( - agencies, - feed_info, - container['trips_store'], - container['stops_store'], - region.bbox) - exporter.export(f"data/gtfs/amarillo.{region.id}.gtfs.zip", "data/tmp/") - -def generate_gtfs_rt(): - logger.info("Generate GTFS-RT") - producer = GtfsRtProducer(container['trips_store']) - for region in regions.values(): - rt = producer.export_feed(time.time(), f"data/gtfs/amarillo.{region.id}.gtfsrt", bbox=region.bbox) - -def start_schedule(): - schedule.every().day.at("00:00").do(midnight) - schedule.every(60).seconds.do(generate_gtfs_rt) - # Create all feeds once at startup - schedule.run_all() - job_thread = threading.Thread(target=run_schedule, daemon=True) - job_thread.start() \ No newline at end of file diff --git a/enhancer.py b/enhancer.py deleted file mode 100644 index 1bdbb87..0000000 --- a/enhancer.py +++ /dev/null @@ -1,70 +0,0 @@ -import json -import time -import logging -import logging.config -from watchdog.observers import Observer -from watchdog.events import FileSystemEventHandler - -from amarillo.configuration import configure_enhancer_services -from amarillo.utils.container import container -from amarillo.models.Carpool import Carpool -from amarillo.utils.utils import agency_carpool_ids_from_filename - -logging.config.fileConfig('logging.conf', disable_existing_loggers=False) -logger = logging.getLogger("enhancer") - -logger.info("Hello Enhancer") - -configure_enhancer_services() - -observer = Observer() # Watch Manager - - -class EventHandler(FileSystemEventHandler): - # TODO FG HB should watch for both carpools and agencies - # in data/agency, data/agencyconf, see AgencyConfService - - def on_closed(self, event): - - logger.info("CLOSE_WRITE: Created %s", event.src_path) - try: - with open(event.src_path, 'r', encoding='utf-8') as f: - dict = json.load(f) - carpool = Carpool(**dict) - - container['carpools'].put(carpool.agency, carpool.id, carpool) - except FileNotFoundError as e: - logger.error("Carpool could not be added, as already deleted (%s)", event.src_path) - except: - logger.exception("Eventhandler on_closed encountered exception") - - def on_deleted(self, event): - try: - logger.info("DELETE: Removing %s", event.src_path) - (agency_id, carpool_id) = agency_carpool_ids_from_filename(event.src_path) - container['carpools'].delete(agency_id, carpool_id) - except: - logger.exception("Eventhandler on_deleted encountered exception") - - -observer.schedule(EventHandler(), 'data/carpool', recursive=True) -observer.start() - -import time - -try: - # TODO FG Is this really needed? - cnt = 0 - ENHANCER_LOG_INTERVAL_IN_S = 600 - while True: - if cnt == ENHANCER_LOG_INTERVAL_IN_S: - logger.debug("Currently stored carpool ids: %s", container['carpools'].get_all_ids()) - cnt = 0 - - time.sleep(1) - cnt += 1 -finally: - observer.stop() - observer.join() - - logger.info("Goodbye Enhancer") From d8f7b0b31ee86d347a9cbaad16533a55c413eb37 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 30 Jan 2024 15:09:00 +0100 Subject: [PATCH 10/50] moved 'configure_enhancer_services' to enhancer plugin --- amarillo/configuration.py | 43 --------------------------------------- 1 file changed, 43 deletions(-) diff --git a/amarillo/configuration.py b/amarillo/configuration.py index 669c988..019508f 100644 --- a/amarillo/configuration.py +++ b/amarillo/configuration.py @@ -1,21 +1,14 @@ # separate file so that it can be imported without initializing FastAPI from amarillo.utils.container import container -import json import logging -from glob import glob -from amarillo.models.Carpool import Agency, Carpool, Region -from amarillo.services import stops -from amarillo.services import trips from amarillo.services.agencyconf import AgencyConfService, agency_conf_directory -from amarillo.services.carpools import CarpoolService from amarillo.services.agencies import AgencyService from amarillo.services.regions import RegionService from amarillo.services.config import config from amarillo.utils.utils import assert_folder_exists -import amarillo.services.gtfs_generator as gtfs_generator logger = logging.getLogger(__name__) @@ -49,42 +42,6 @@ def configure_services(): create_required_directories() - -def configure_enhancer_services(): - configure_services() - - logger.info("Load stops...") - with open(config.stop_sources_file) as stop_sources_file: - stop_sources = json.load(stop_sources_file) - stop_store = stops.StopsStore(stop_sources) - - stop_store.load_stop_sources() - container['stops_store'] = stop_store - container['trips_store'] = trips.TripStore(stop_store) - container['carpools'] = CarpoolService(container['trips_store']) - - logger.info("Restore carpools...") - - for agency_id in container['agencies'].agencies: - for carpool_file_name in glob(f'data/carpool/{agency_id}/*.json'): - try: - with open(carpool_file_name) as carpool_file: - carpool = Carpool(**(json.load(carpool_file))) - container['carpools'].put(carpool.agency, carpool.id, carpool) - except Exception as e: - logger.warning("Issue during restore of carpool %s: %s", carpool_file_name, repr(e)) - - # notify carpool about carpools in trash, as delete notifications must be sent - for carpool_file_name in glob(f'data/trash/{agency_id}/*.json'): - with open(carpool_file_name) as carpool_file: - carpool = Carpool(**(json.load(carpool_file))) - container['carpools'].delete(carpool.agency, carpool.id) - - logger.info("Restored carpools: %s", container['carpools'].get_all_ids()) - logger.info("Starting scheduler") - gtfs_generator.start_schedule() - - def configure_admin_token(): if config.admin_token is None: message = "ADMIN_TOKEN environment variable not set" From 25c76aec58c2bb446c6f9dea127b70d948c58e7d Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 6 Feb 2024 11:38:19 +0100 Subject: [PATCH 11/50] moved routing.py, trips.py and stops.py to enhancer plugin --- amarillo/services/routing.py | 47 ----- amarillo/services/stops.py | 131 ------------ amarillo/services/trips.py | 375 ----------------------------------- 3 files changed, 553 deletions(-) delete mode 100644 amarillo/services/routing.py delete mode 100644 amarillo/services/stops.py delete mode 100644 amarillo/services/trips.py diff --git a/amarillo/services/routing.py b/amarillo/services/routing.py deleted file mode 100644 index 96f6229..0000000 --- a/amarillo/services/routing.py +++ /dev/null @@ -1,47 +0,0 @@ -import requests -import logging - -logger = logging.getLogger(__name__) - -class RoutingException(Exception): - def __init__(self, message): - # Call Exception.__init__(message) - # to use the same Message header as the parent class - super().__init__(message) - -class RoutingService(): - def __init__(self, gh_url = 'https://api.mfdz.de/gh'): - self.gh_service_url = gh_url - - def path_for_stops(self, points): - # Retrieve graphhopper route traversing given points - directions = self._get_directions(points) - if directions and len(directions.get("paths"))>0: - return directions.get("paths")[0] - else: - return {} - - def _get_directions(self, points): - req_url = self._create_url(points, True, True) - logger.debug("Get directions via: {}".format(req_url)) - response = requests.get(req_url) - status = response.status_code - if status == 200: - # Found route between points - return response.json() - else: - try: - message = response.json().get('message') - except: - raise RoutingException("Get directions failed with status code {}".format(status)) - else: - raise RoutingException(message) - - def _create_url(self, points, calc_points = False, instructions = False): - """ Creates GH request URL """ - locations = "" - for point in points: - locations += "point={0}%2C{1}&".format(point.y, point.x) - - return "{0}/route?{1}instructions={2}&calc_points={3}&points_encoded=false&profile=car".format( - self.gh_service_url, locations, instructions, calc_points) diff --git a/amarillo/services/stops.py b/amarillo/services/stops.py deleted file mode 100644 index 1c00791..0000000 --- a/amarillo/services/stops.py +++ /dev/null @@ -1,131 +0,0 @@ -import codecs -import csv -import logging -import re -from contextlib import closing -from io import TextIOWrapper - -import geopandas as gpd -import pandas as pd -import requests -from pyproj import Proj, Transformer -from shapely.geometry import LineString, Point -from shapely.ops import transform - -from amarillo.models.Carpool import StopTime - -from .stop_importer import CsvStopsImporter, GeojsonStopsImporter, GtfsStopsImporter, OverpassStopsImporter - -logger = logging.getLogger(__name__) - - -def is_carpooling_stop(stop_id, name): - stop_name = name.lower() - # mfdz: or bbnavi: prefixed stops are custom stops which are explicitly meant to be carpooling stops - return stop_id.startswith('mfdz:') or stop_id.startswith('bbnavi:') or 'mitfahr' in stop_name or 'p&m' in stop_name - - -class StopsStore: - def __init__(self, stop_sources=None, internal_projection='EPSG:32632'): - self.internal_projection = internal_projection - self.projection = Transformer.from_crs('EPSG:4326', internal_projection, always_xy=True).transform - self.stopsDataFrames = [] - self.stop_sources = stop_sources if stop_sources is not None else [] - - def load_stop_sources(self): - """Imports stops from stop_sources and registers them with - the distance they are still associated with a trip. - E.g. bus stops should be registered with a distance of e.g. 30m, - while larger carpool parkings might be registered with e.g. 500m. - - Subsequent calls of load_stop_sources will reload all stop_sources - but replace the current stops only if all stops could be loaded successfully. - """ - stopsDataFrames = [] - error_occured = False - - for stops_source in self.stop_sources: - try: - source_url = stops_source.get('url') - source_type = stops_source.get('type') or ( - 'geojson' - if source_url is not None and source_url.startswith('http') and source_url.endswith('json') - else 'csv' - ) - logger.info('Loading stop source %s...', stops_source.get('id')) - match source_type: - case 'geojson': - stopsDataFrame = GeojsonStopsImporter().load_stops(source_url) - case 'csv': - stopsDataFrame = CsvStopsImporter().load_stops(source_url) - case 'overpass': - stopsDataFrame = OverpassStopsImporter().load_stops(**stops_source) - case 'gtfs': - stopsDataFrame = GtfsStopsImporter().load_stops(**stops_source) - case _: - logger.error('Failed to load stops, source type %s not supported', source_type) - continue - stopsDataFrame.to_crs(crs=self.internal_projection, inplace=True) - stopsDataFrames.append({'distanceInMeter': stops_source['vicinity'], 'stops': stopsDataFrame}) - except Exception: - error_occured = True - logger.error('Failed to load stops from %s to StopsStore.', stops_source, exc_info=True) - - if not error_occured: - self.stopsDataFrames = stopsDataFrames - - def find_additional_stops_around(self, line, stops=None): - """Returns a GeoDataFrame with all stops in vicinity of the - given line, sorted by distance from origin of the line. - Note: for internal projection/distance calculations, the - lat/lon geometries of line and stops are converted to - """ - stops_frames = [] - if stops: - stops_frames.append(self._convert_to_dataframe(stops)) - transformedLine = transform(self.projection, LineString(line.coordinates)) - for stops_to_match in self.stopsDataFrames: - stops_frames.append( - self._find_stops_around_transformed( - stops_to_match['stops'], transformedLine, stops_to_match['distanceInMeter'] - ) - ) - stops = gpd.GeoDataFrame(pd.concat(stops_frames, ignore_index=True, sort=True)) - if not stops.empty: - self._sort_by_distance(stops, transformedLine) - return stops - - def find_closest_stop(self, carpool_stop, max_search_distance): - transformedCoord = Point(self.projection(carpool_stop.lon, carpool_stop.lat)) - best_dist = max_search_distance + 1 - best_stop = None - for stops_with_dist in self.stopsDataFrames: - stops = stops_with_dist['stops'] - s, d = stops.sindex.nearest( - transformedCoord, return_all=True, return_distance=True, max_distance=max_search_distance - ) - if len(d) > 0 and d[0] < best_dist: - best_dist = d[0] - row = s[1][0] - best_stop = StopTime(name=stops.at[row, 'stop_name'], lat=stops.at[row, 'y'], lon=stops.at[row, 'x']) - - return best_stop if best_stop else carpool_stop - - def _find_stops_around_transformed(self, stopsDataFrame, transformedLine, distance): - bufferedLine = transformedLine.buffer(distance) - sindex = stopsDataFrame.sindex - possible_matches_index = list(sindex.intersection(bufferedLine.bounds)) - possible_matches = stopsDataFrame.iloc[possible_matches_index] - - return possible_matches[possible_matches.intersects(bufferedLine)] - - def _convert_to_dataframe(self, stops): - return gpd.GeoDataFrame( - [[stop.name, stop.lon, stop.lat, stop.id, Point(self.projection(stop.lon, stop.lat))] for stop in stops], - columns=['stop_name', 'x', 'y', 'id', 'geometry'], - crs=self.internal_projection, - ) - - def _sort_by_distance(self, stops, transformedLine): - stops['distance'] = stops.apply(lambda row: transformedLine.project(row['geometry']), axis=1) - stops.sort_values('distance', inplace=True) diff --git a/amarillo/services/trips.py b/amarillo/services/trips.py deleted file mode 100644 index fcbc21b..0000000 --- a/amarillo/services/trips.py +++ /dev/null @@ -1,375 +0,0 @@ -from amarillo.services.config import config -from amarillo.models.gtfs import GtfsTimeDelta, GtfsStopTime -from amarillo.models.Carpool import MAX_STOPS_PER_TRIP, Carpool, Weekday, StopTime, PickupDropoffType -from amarillo.services.gtfs_constants import * -from amarillo.services.routing import RoutingService, RoutingException -from amarillo.services.stops import is_carpooling_stop -from amarillo.utils.utils import assert_folder_exists, is_older_than_days, yesterday, geodesic_distance_in_m -from shapely.geometry import Point, LineString, box -from geojson_pydantic.geometries import LineString as GeoJSONLineString -from datetime import datetime, timedelta -import numpy as np -import os -import json -import logging - -logger = logging.getLogger(__name__) - -class Trip: - - def __init__(self, trip_id, route_name, headsign, url, calendar, departureTime, path, agency, lastUpdated, stop_times, bbox): - if isinstance(calendar, set): - self.runs_regularly = True - self.weekdays = [ - 1 if Weekday.monday in calendar else 0, - 1 if Weekday.tuesday in calendar else 0, - 1 if Weekday.wednesday in calendar else 0, - 1 if Weekday.thursday in calendar else 0, - 1 if Weekday.friday in calendar else 0, - 1 if Weekday.saturday in calendar else 0, - 1 if Weekday.sunday in calendar else 0, - ] - start_in_day = self._total_seconds(departureTime) - else: - self.start = datetime.combine(calendar, departureTime) - self.runs_regularly = False - self.weekdays = [0,0,0,0,0,0,0] - - self.start_time = departureTime - self.path = path - self.trip_id = trip_id - self.url = url - self.agency = agency - self.stops = [] - self.lastUpdated = lastUpdated - self.stop_times = stop_times - self.bbox = bbox - self.route_name = route_name - self.trip_headsign = headsign - - def path_as_line_string(self): - return path - - def _total_seconds(self, instant): - return instant.hour * 3600 + instant.minute * 60 + instant.second - - def start_time_str(self): - return self.start_time.strftime("%H:%M:%S") - - def next_trip_dates(self, start_date, day_count=14): - if self.runs_regularly: - for single_date in (start_date + timedelta(n) for n in range(day_count)): - if self.weekdays[single_date.weekday()]==1: - yield single_date.strftime("%Y%m%d") - else: - yield self.start.strftime("%Y%m%d") - - def route_long_name(self): - return self.route_name - - def intersects(self, bbox): - return self.bbox.intersects(box(*bbox)) - - -class TripStore(): - """ - TripStore maintains the currently valid trips. A trip is a - carpool offer enhanced with all stops this - - Attributes: - trips Dict of currently valid trips. - deleted_trips Dict of recently deleted trips. - """ - - def __init__(self, stops_store): - self.transformer = TripTransformer(stops_store) - self.stops_store = stops_store - self.trips = {} - self.deleted_trips = {} - self.recent_trips = {} - - - def put_carpool(self, carpool: Carpool): - """ - Adds carpool to the TripStore. - """ - id = "{}:{}".format(carpool.agency, carpool.id) - filename = f'data/enhanced/{carpool.agency}/{carpool.id}.json' - try: - existing_carpool = self._load_carpool_if_exists(carpool.agency, carpool.id) - if existing_carpool and existing_carpool.lastUpdated == carpool.lastUpdated: - enhanced_carpool = existing_carpool - else: - if len(carpool.stops) < 2 or self.distance_in_m(carpool) < 1000: - logger.warning("Failed to add carpool %s:%s to TripStore, distance too low", carpool.agency, carpool.id) - self.handle_failed_carpool_enhancement(carpool) - return - enhanced_carpool = self.transformer.enhance_carpool(carpool) - # TODO should only store enhanced_carpool, if it has 2 or more stops - assert_folder_exists(f'data/enhanced/{carpool.agency}/') - with open(filename, 'w', encoding='utf-8') as f: - f.write(enhanced_carpool.json()) - logger.info("Added enhanced carpool %s:%s", carpool.agency, carpool.id) - - return self._load_as_trip(enhanced_carpool) - except RoutingException as err: - logger.warning("Failed to add carpool %s:%s to TripStore due to RoutingException %s", carpool.agency, carpool.id, getattr(err, 'message', repr(err))) - self.handle_failed_carpool_enhancement(carpool) - except Exception as err: - logger.error("Failed to add carpool %s:%s to TripStore.", carpool.agency, carpool.id, exc_info=True) - self.handle_failed_carpool_enhancement(carpool) - - def handle_failed_carpool_enhancement(sellf, carpool: Carpool): - assert_folder_exists(f'data/failed/{carpool.agency}/') - with open(f'data/failed/{carpool.agency}/{carpool.id}.json', 'w', encoding='utf-8') as f: - f.write(carpool.json()) - - def distance_in_m(self, carpool): - if len(carpool.stops) < 2: - return 0 - s1 = carpool.stops[0] - s2 = carpool.stops[-1] - return geodesic_distance_in_m((s1.lon, s1.lat),(s2.lon, s2.lat)) - - def recently_added_trips(self): - return list(self.recent_trips.values()) - - def recently_deleted_trips(self): - return list(self.deleted_trips.values()) - - def _load_carpool_if_exists(self, agency_id: str, carpool_id: str): - if carpool_exists(agency_id, carpool_id, 'data/enhanced'): - try: - return load_carpool(agency_id, carpool_id, 'data/enhanced') - except Exception as e: - # An error on restore could be caused by model changes, - # in such a case, it need's to be recreated - logger.warning("Could not restore enhanced trip %s:%s, reason: %s", agency_id, carpool_id, repr(e)) - - return None - - def _load_as_trip(self, carpool: Carpool): - trip = self.transformer.transform_to_trip(carpool) - id = trip.trip_id - self.trips[id] = trip - if not is_older_than_days(carpool.lastUpdated, 1): - self.recent_trips[id] = trip - logger.debug("Added trip %s", id) - - return trip - - def delete_carpool(self, agency_id: str, carpool_id: str): - """ - Deletes carpool from the TripStore. - """ - agencyScopedCarpoolId = f"{agency_id}:{carpool_id}" - trip_to_be_deleted = self.trips.get(agencyScopedCarpoolId) - if trip_to_be_deleted: - self.deleted_trips[agencyScopedCarpoolId] = trip_to_be_deleted - del self.trips[agencyScopedCarpoolId] - - if self.recent_trips.get(agencyScopedCarpoolId): - del self.recent_trips[agencyScopedCarpoolId] - - if carpool_exists(agency_id, carpool_id): - remove_carpool_file(agency_id, carpool_id) - - logger.debug("Deleted trip %s", id) - - def unflag_unrecent_updates(self): - """ - Trips that were last updated before yesterday, are not recent - any longer. As no updates need to be sent for them any longer, - they will be removed from recent recent_trips and deleted_trips. - """ - for key in list(self.recent_trips): - t = self.recent_trips.get(key) - if t and t.lastUpdated.date() < yesterday(): - del self.recent_trips[key] - - for key in list(self.deleted_trips): - t = self.deleted_trips.get(key) - if t and t.lastUpdated.date() < yesterday(): - del self.deleted_trips[key] - - -class TripTransformer: - REPLACE_CARPOOL_STOPS_BY_CLOSEST_TRANSIT_STOPS = True - REPLACEMENT_STOPS_SERACH_RADIUS_IN_M = 1000 - SIMPLIFY_TOLERANCE = 0.0001 - - router = RoutingService(config.graphhopper_base_url) - - def __init__(self, stops_store): - self.stops_store = stops_store - - def transform_to_trip(self, carpool): - stop_times = self._convert_stop_times(carpool) - route_name = carpool.stops[0].name + " nach " + carpool.stops[-1].name - headsign= carpool.stops[-1].name - trip_id = self._trip_id(carpool) - path = carpool.path - bbox = box( - min([pt[0] for pt in path.coordinates]), - min([pt[1] for pt in path.coordinates]), - max([pt[0] for pt in path.coordinates]), - max([pt[1] for pt in path.coordinates])) - - trip = Trip(trip_id, route_name, headsign, str(carpool.deeplink), carpool.departureDate, carpool.departureTime, carpool.path, carpool.agency, carpool.lastUpdated, stop_times, bbox) - - return trip - - def _trip_id(self, carpool): - return f"{carpool.agency}:{carpool.id}" - - def _replace_stops_by_transit_stops(self, carpool, max_search_distance): - new_stops = [] - for carpool_stop in carpool.stops: - new_stops.append(self.stops_store.find_closest_stop(carpool_stop, max_search_distance)) - return new_stops - - def enhance_carpool(self, carpool): - if self.REPLACE_CARPOOL_STOPS_BY_CLOSEST_TRANSIT_STOPS: - carpool.stops = self._replace_stops_by_transit_stops(carpool, self.REPLACEMENT_STOPS_SERACH_RADIUS_IN_M) - - path = self._path_for_ride(carpool) - lineString_shapely_wgs84 = LineString(coordinates = path["points"]["coordinates"]).simplify(0.0001) - lineString_wgs84 = GeoJSONLineString(type="LineString", coordinates=list(lineString_shapely_wgs84.coords)) - virtual_stops = self.stops_store.find_additional_stops_around(lineString_wgs84, carpool.stops) - if not virtual_stops.empty: - virtual_stops["time"] = self._estimate_times(path, virtual_stops['distance']) - logger.debug("Virtual stops found: {}".format(virtual_stops)) - if len(virtual_stops) > MAX_STOPS_PER_TRIP: - # in case we found more than MAX_STOPS_PER_TRIP, we retain first and last - # half of MAX_STOPS_PER_TRIP - virtual_stops = virtual_stops.iloc[np.r_[0:int(MAX_STOPS_PER_TRIP/2), int(MAX_STOPS_PER_TRIP/2):]] - - trip_id = f"{carpool.agency}:{carpool.id}" - stop_times = self._stops_and_stop_times(carpool.departureTime, trip_id, virtual_stops) - - enhanced_carpool = carpool.copy() - enhanced_carpool.stops = stop_times - enhanced_carpool.path = lineString_wgs84 - return enhanced_carpool - - def _convert_stop_times(self, carpool): - - stop_times = [GtfsStopTime( - self._trip_id(carpool), - stop.arrivalTime, - stop.departureTime, - stop.id, - seq_nr+1, - STOP_TIMES_STOP_TYPE_NONE if stop.pickup_dropoff == PickupDropoffType.only_dropoff else STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER, - STOP_TIMES_STOP_TYPE_NONE if stop.pickup_dropoff == PickupDropoffType.only_pickup else STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER, - STOP_TIMES_TIMEPOINT_APPROXIMATE) - for seq_nr, stop in enumerate(carpool.stops)] - return stop_times - - def _path_for_ride(self, carpool): - points = self._stop_coords(carpool.stops) - return self.router.path_for_stops(points) - - def _stop_coords(self, stops): - # Retrieve coordinates of all officially announced stops (start, intermediate, target) - return [Point(stop.lon, stop.lat) for stop in stops] - - def _estimate_times(self, path, distances_from_start): - cumulated_distance = 0 - cumulated_time = 0 - stop_times = [] - instructions = path["instructions"] - - cnt = 0 - instr_distance = instructions[cnt]["distance"] - instr_time = instructions[cnt]["time"] - - for distance in distances_from_start: - while cnt < len(instructions) and cumulated_distance + instructions[cnt]["distance"] < distance: - cumulated_distance = cumulated_distance + instructions[cnt]["distance"] - cumulated_time = cumulated_time + instructions[cnt]["time"] - cnt = cnt + 1 - - if cnt < len(instructions): - if instructions[cnt]["distance"] ==0: - raise RoutingException("Origin and destinaction too close") - percent_dist = (distance - cumulated_distance) / instructions[cnt]["distance"] - stop_time = cumulated_time + percent_dist * instructions[cnt]["time"] - stop_times.append(stop_time) - else: - logger.debug("distance {} exceeds total length {}, using max arrival time {}".format(distance, cumulated_distance, cumulated_time)) - stop_times.append(cumulated_time) - return stop_times - - def _stops_and_stop_times(self, start_time, trip_id, stops_frame): - # Assumptions: - # arrival_time = departure_time - # pickup_type, drop_off_type for origin: = coordinate/none - # pickup_type, drop_off_type for destination: = none/coordinate - # timepoint = approximate for origin and destination (not sure what consequences this might have for trip planners) - number_of_stops = len(stops_frame.index) - total_distance = stops_frame.iloc[number_of_stops-1]["distance"] - - first_stop_time = GtfsTimeDelta(hours = start_time.hour, minutes = start_time.minute, seconds = start_time.second) - stop_times = [] - seq_nr = 0 - for i in range(0, number_of_stops): - current_stop = stops_frame.iloc[i] - - if not current_stop.id: - continue - elif i == 0: - if (stops_frame.iloc[1].time-current_stop.time) < 1000: - # skip custom stop if there is an official stop very close by - logger.debug("Skipped stop %s", current_stop.id) - continue - else: - if (current_stop.time-stops_frame.iloc[i-1].time) < 5000 and not i==1 and not is_carpooling_stop(current_stop.id, current_stop.stop_name): - # skip latter stop if it's very close (<5 seconds drive) by the preceding - logger.debug("Skipped stop %s", current_stop.id) - continue - trip_time = timedelta(milliseconds=int(current_stop.time)) - is_dropoff = self._is_dropoff_stop(current_stop, total_distance) - is_pickup = self._is_pickup_stop(current_stop, total_distance) - # TODO would be nice if possible to publish a minimum shared distance - pickup_type = STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER if is_pickup else STOP_TIMES_STOP_TYPE_NONE - dropoff_type = STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER if is_dropoff else STOP_TIMES_STOP_TYPE_NONE - - if is_pickup and not is_dropoff: - pickup_dropoff = PickupDropoffType.only_pickup - elif not is_pickup and is_dropoff: - pickup_dropoff = PickupDropoffType.only_dropoff - else: - pickup_dropoff = PickupDropoffType.pickup_and_dropoff - - next_stop_time = first_stop_time + trip_time - seq_nr += 1 - stop_times.append(StopTime(**{ - 'arrivalTime': str(next_stop_time), - 'departureTime': str(next_stop_time), - 'id': current_stop.id, - 'pickup_dropoff': pickup_dropoff, - 'name': str(current_stop.stop_name), - 'lat': current_stop.y, - 'lon': current_stop.x - })) - - return stop_times - - def _is_dropoff_stop(self, current_stop, total_distance): - return current_stop["distance"] >= 0.5 * total_distance - - def _is_pickup_stop(self, current_stop, total_distance): - return current_stop["distance"] < 0.5 * total_distance - -def load_carpool(agency_id: str, carpool_id: str, folder: str ='data/enhanced') -> Carpool: - with open(f'{folder}/{agency_id}/{carpool_id}.json', 'r', encoding='utf-8') as f: - dict = json.load(f) - carpool = Carpool(**dict) - return carpool - -def carpool_exists(agency_id: str, carpool_id: str, folder: str ='data/enhanced'): - return os.path.exists(f"{folder}/{agency_id}/{carpool_id}.json") - -def remove_carpool_file(agency_id: str, carpool_id: str, folder: str ='data/enhanced'): - return os.remove(f"{folder}/{agency_id}/{carpool_id}.json") From 1e6cea89f783a6cc126cce74bfe54d61cea0806a Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 6 Feb 2024 15:11:48 +0100 Subject: [PATCH 12/50] removed enhancer from Dockerfile --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d1babed..aa563ca 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,8 +29,6 @@ RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt ${PLUGINS} ENV MODULE_NAME=amarillo.main COPY ./amarillo /app/amarillo -COPY enhancer.py /app -COPY prestart.sh /app COPY ./amarillo/plugins /app/amarillo/plugins COPY ./amarillo/static/static /app/static COPY ./amarillo/static/templates /app/templates From cc493c3cfc4bf42676fda595328d55297f6e02e7 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 6 Feb 2024 15:13:43 +0100 Subject: [PATCH 13/50] moved tests to enhancer --- amarillo/tests/stops.csv | 5 - amarillo/tests/stops.json | 39 -------- amarillo/tests/test_gtfs.py | 142 ----------------------------- amarillo/tests/test_stops_store.py | 28 ------ amarillo/tests/test_trip_store.py | 23 ----- 5 files changed, 237 deletions(-) delete mode 100644 amarillo/tests/stops.csv delete mode 100644 amarillo/tests/stops.json delete mode 100644 amarillo/tests/test_gtfs.py delete mode 100644 amarillo/tests/test_stops_store.py delete mode 100644 amarillo/tests/test_trip_store.py diff --git a/amarillo/tests/stops.csv b/amarillo/tests/stops.csv deleted file mode 100644 index ed419bb..0000000 --- a/amarillo/tests/stops.csv +++ /dev/null @@ -1,5 +0,0 @@ -stop_id;stop_code;stop_lat;stop_lon;stop_name -mfdz:x;x;52.11901;14.2;Stop x -mfdz:y;y;53.1;14.01;Stop y -mfdz:z;z;54.11;14.0;Stop z -mfdz:Ang001;Ang001;53.11901;14.015776;Mitfahrbank Biesenbrow diff --git a/amarillo/tests/stops.json b/amarillo/tests/stops.json deleted file mode 100644 index 5744c13..0000000 --- a/amarillo/tests/stops.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "data": { - "pointsOfInterest": [ - { - "id": "14622", - "externalId": "bbnavi:12073:0001", - "name": "Parkbank", - "description": "Parkbank", - "dataProvider": { - "id": "1", - "name": "Administrator" - }, - "addresses": [ - { - "street": "Hauptstrasse", - "city": "Wittenberge", - "zip": "12345", - "geoLocation": { - "latitude": 52.9932971109789, - "longitude": 11.767383582547 - } - } - ], - "openStreetMap": { - "capacity": 112, - "capacityCharging": "2", - "capacityDisabled": "", - "fee": "No", - "lit": "Yes", - "parking": "", - "shelter": "No", - "surface": "", - "utilization": "", - "website": "" - } - } - ] - } -} \ No newline at end of file diff --git a/amarillo/tests/test_gtfs.py b/amarillo/tests/test_gtfs.py deleted file mode 100644 index 61a8e5a..0000000 --- a/amarillo/tests/test_gtfs.py +++ /dev/null @@ -1,142 +0,0 @@ -from amarillo.tests.sampledata import carpool_1234, data1, carpool_repeating_json, stop_issue -from amarillo.services.gtfs_export import GtfsExport -from amarillo.services.gtfs import GtfsRtProducer -from amarillo.services.stops import StopsStore -from amarillo.services.trips import TripStore -from amarillo.models.Carpool import Carpool -from datetime import datetime -import time -import pytest - - -def test_gtfs_generation(): - cp = Carpool(**data1) - stops_store = StopsStore() - trips_store = TripStore(stops_store) - trips_store.put_carpool(cp) - - exporter = GtfsExport(None, None, trips_store, stops_store) - exporter.export('target/tests/test_gtfs_generation/test.gtfs.zip', "target/tests/test_gtfs_generation") - -def test_correct_stops(): - cp = Carpool(**stop_issue) - stops_store = StopsStore([{"url": "https://datahub.bbnavi.de/export/rideshare_points.geojson", "vicinity": 250}]) - stops_store.load_stop_sources() - trips_store = TripStore(stops_store) - trips_store.put_carpool(cp) - assert len(trips_store.trips) == 1 - - -class TestTripConverter: - - def setup_method(self, method): - self.stops_store = StopsStore([{"url": "https://datahub.bbnavi.de/export/rideshare_points.geojson", "vicinity": 50}]) - self.trips_store = TripStore(self.stops_store) - - def test_as_one_time_trip_as_delete_update(self): - cp = Carpool(**data1) - self.trips_store.put_carpool(cp) - trip = next(iter(self.trips_store.trips.values())) - - converter = GtfsRtProducer(self.trips_store) - json = converter._as_delete_updates(trip, datetime(2022,4,11)) - - assert json == [{ - 'trip': { - 'tripId': 'mfdz:Eins', - 'startTime': '23:59:00', - 'startDate': '20220530', - 'scheduleRelationship': 'CANCELED', - 'routeId': 'mfdz:Eins' - } - }] - - def test_as_one_time_trip_as_added_update(self): - cp = Carpool(**data1) - self.trips_store.put_carpool(cp) - trip = next(iter(self.trips_store.trips.values())) - - converter = GtfsRtProducer(self.trips_store) - json = converter._as_added_updates(trip, datetime(2022,4,11)) - assert json == [{ - 'trip': { - 'tripId': 'mfdz:Eins', - 'startTime': '23:59:00', - 'startDate': '20220530', - 'scheduleRelationship': 'ADDED', - 'routeId': 'mfdz:Eins', - '[transit_realtime.trip_descriptor]': { - 'routeUrl' : 'https://mfdz.de/trip/123', - 'agencyId' : 'mfdz', - 'route_long_name' : 'abc nach xyz', - 'route_type': 1551 - } - }, - 'stopTimeUpdate': [{ - 'stopSequence': 1, - 'arrival': { - 'time': time.mktime(datetime(2022,5,30,23,59,0).timetuple()), - 'uncertainty': 600 - }, - 'departure': { - 'time': time.mktime(datetime(2022,5,30,23,59,0).timetuple()), - 'uncertainty': 600 - }, - 'stopId': 'mfdz:12073:001', - 'scheduleRelationship': 'SCHEDULED', - 'stop_time_properties': { - '[transit_realtime.stop_time_properties]': { - 'dropoffType': 'NONE', - 'pickupType': 'COORDINATE_WITH_DRIVER' - } - } - }, - { - 'stopSequence': 2, - 'arrival': { - 'time': time.mktime(datetime(2022,5,31,0,16,45,0).timetuple()), - 'uncertainty': 600 - }, - 'departure': { - 'time': time.mktime(datetime(2022,5,31,0,16,45,0).timetuple()), - 'uncertainty': 600 - }, - - 'stopId': 'de:12073:900340137::3', - 'scheduleRelationship': 'SCHEDULED', - 'stop_time_properties': { - '[transit_realtime.stop_time_properties]': { - 'dropoffType': 'COORDINATE_WITH_DRIVER', - 'pickupType': 'NONE' - } - } - }] - }] - - def test_as_periodic_trip_as_delete_update(self): - cp = Carpool(**carpool_repeating_json) - self.trips_store.put_carpool(cp) - trip = next(iter(self.trips_store.trips.values())) - - converter = GtfsRtProducer(self.trips_store) - json = converter._as_delete_updates(trip, datetime(2022,4,11)) - - assert json == [{ - 'trip': { - 'tripId': 'mfdz:Zwei', - 'startTime': '15:00:00', - 'startDate': '20220411', - 'scheduleRelationship': 'CANCELED', - 'routeId': 'mfdz:Zwei' - } - }, - { - 'trip': { - 'tripId': 'mfdz:Zwei', - 'startTime': '15:00:00', - 'startDate': '20220418', - 'scheduleRelationship': 'CANCELED', - 'routeId': 'mfdz:Zwei' - } - } - ] \ No newline at end of file diff --git a/amarillo/tests/test_stops_store.py b/amarillo/tests/test_stops_store.py deleted file mode 100644 index 931a562..0000000 --- a/amarillo/tests/test_stops_store.py +++ /dev/null @@ -1,28 +0,0 @@ -from amarillo.models.Carpool import StopTime -from amarillo.services import stops - - -def test_load_stops_from_file(): - store = stops.StopsStore([{'url': 'amarillo/tests/stops.csv', 'vicinity': 50}]) - store.load_stop_sources() - assert len(store.stopsDataFrames[0]['stops']) > 0 - - -def test_load_csv_stops_from_web_(): - store = stops.StopsStore([{'url': 'https://data.mfdz.de/mfdz/stops/custom.csv', 'vicinity': 50}]) - store.load_stop_sources() - assert len(store.stopsDataFrames[0]['stops']) > 0 - - -def test_load_geojson_stops_from_web_(): - store = stops.StopsStore([{'url': 'https://datahub.bbnavi.de/export/rideshare_points.geojson', 'vicinity': 50}]) - store.load_stop_sources() - assert len(store.stopsDataFrames[0]['stops']) > 0 - - -def test_find_closest_stop(): - store = stops.StopsStore([{'url': 'amarillo/tests/stops.csv', 'vicinity': 50}]) - store.load_stop_sources() - carpool_stop = StopTime(name='start', lat=53.1191, lon=14.01577) - stop = store.find_closest_stop(carpool_stop, 1000) - assert stop.name == 'Mitfahrbank Biesenbrow' diff --git a/amarillo/tests/test_trip_store.py b/amarillo/tests/test_trip_store.py deleted file mode 100644 index 96c9616..0000000 --- a/amarillo/tests/test_trip_store.py +++ /dev/null @@ -1,23 +0,0 @@ -from amarillo.tests.sampledata import cp1, carpool_repeating -from amarillo.services.trips import TripStore -from amarillo.services.stops import StopsStore - - -import logging -logger = logging.getLogger(__name__) - -def test_trip_store_put_one_time_carpool(): - trip_store = TripStore(StopsStore()) - - t = trip_store.put_carpool(cp1) - assert t != None - assert len(t.stop_times) >= 2 - assert t.stop_times[0].stop_id == 'mfdz:12073:001' - assert t.stop_times[-1].stop_id == 'de:12073:900340137::3' - -def test_trip_store_put_repeating_carpool(): - trip_store = TripStore(StopsStore()) - - t = trip_store.put_carpool(carpool_repeating) - assert t != None - assert len(t.stop_times) >= 2 From 8ff0d1fc519c6e8c355c5d50b8ee5b3454ae5f97 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 6 Feb 2024 15:32:14 +0100 Subject: [PATCH 14/50] moved gtfs services to enhancer --- amarillo/services/gtfs.py | 137 ------------------ amarillo/services/gtfs_constants.py | 14 -- amarillo/services/gtfsrt/__init__.py | 0 amarillo/services/gtfsrt/gtfs_realtime_pb2.py | 80 ---------- .../services/gtfsrt/realtime_extension_pb2.py | 33 ----- 5 files changed, 264 deletions(-) delete mode 100644 amarillo/services/gtfs.py delete mode 100644 amarillo/services/gtfs_constants.py delete mode 100644 amarillo/services/gtfsrt/__init__.py delete mode 100644 amarillo/services/gtfsrt/gtfs_realtime_pb2.py delete mode 100644 amarillo/services/gtfsrt/realtime_extension_pb2.py diff --git a/amarillo/services/gtfs.py b/amarillo/services/gtfs.py deleted file mode 100644 index c45a626..0000000 --- a/amarillo/services/gtfs.py +++ /dev/null @@ -1,137 +0,0 @@ -import amarillo.services.gtfsrt.gtfs_realtime_pb2 as gtfs_realtime_pb2 -import amarillo.services.gtfsrt.realtime_extension_pb2 as mfdzrte -from amarillo.services.gtfs_constants import * -from google.protobuf.json_format import MessageToDict -from google.protobuf.json_format import ParseDict -from datetime import datetime, timedelta -import json -import re -import time - -class GtfsRtProducer(): - - def __init__(self, trip_store): - self.trip_store = trip_store - - def generate_feed(self, time, format='protobuf', bbox=None): - # See https://developers.google.com/transit/gtfs-realtime/reference - # https://github.com/mfdz/carpool-gtfs-rt/blob/master/src/main/java/de/mfdz/resource/CarpoolResource.java - gtfsrt_dict = { - 'header': { - 'gtfsRealtimeVersion': '1.0', - 'timestamp': int(time) - }, - 'entity': self._get_trip_updates(bbox) - } - feed = gtfs_realtime_pb2.FeedMessage() - ParseDict(gtfsrt_dict, feed) - - if "message" == format.lower(): - return feed - elif "json" == format.lower(): - return MessageToDict(feed) - else: - return feed.SerializeToString() - - def export_feed(self, timestamp, file_path, bbox=None): - """ - Exports gtfs-rt feed as .json and .pbf file to file_path - """ - feed = self.generate_feed(timestamp, "message", bbox) - with open(f"{file_path}.pbf", "wb") as f: - f.write(feed.SerializeToString()) - with open(f"{file_path}.json", "w") as f: - json.dump(MessageToDict(feed), f) - - def _get_trip_updates(self, bbox = None): - trips = [] - trips.extend(self._get_added(bbox)) - trips.extend(self._get_deleted(bbox)) - trip_updates = [] - for num, trip in enumerate(trips): - trip_updates.append( { - 'id': f'carpool-update-{num}', - 'tripUpdate': trip - } - ) - return trip_updates - - def _get_deleted(self, bbox = None): - return self._get_updates( - self.trip_store.recently_deleted_trips(), - self._as_delete_updates, - bbox) - - def _get_added(self, bbox = None): - return self._get_updates( - self.trip_store.recently_added_trips(), - self._as_added_updates, - bbox) - - def _get_updates(self, trips, update_func, bbox = None): - updates = [] - today = datetime.today() - for t in trips: - if bbox == None or t.intersects(bbox): - updates.extend(update_func(t, today)) - return updates - - def _as_delete_updates(self, trip, fromdate): - return [{ - 'trip': { - 'tripId': trip.trip_id, - 'startTime': trip.start_time_str(), - 'startDate': trip_date, - 'scheduleRelationship': 'CANCELED', - 'routeId': trip.trip_id - } - } for trip_date in trip.next_trip_dates(fromdate)] - - def _to_seconds(self, fromdate, stop_time): - startdate = datetime.strptime(fromdate, '%Y%m%d') - m = re.search(r'(\d+):(\d+):(\d+)', stop_time) - delta = timedelta( - hours=int(m.group(1)), - minutes=int(m.group(2)), - seconds=int(m.group(3))) - return time.mktime((startdate + delta).timetuple()) - - def _to_stop_times(self, trip, fromdate): - return [{ - 'stopSequence': stoptime.stop_sequence, - 'arrival': { - 'time': self._to_seconds(fromdate, stoptime.arrival_time), - 'uncertainty': MFDZ_DEFAULT_UNCERTAINITY - }, - 'departure': { - 'time': self._to_seconds(fromdate, stoptime.departure_time), - 'uncertainty': MFDZ_DEFAULT_UNCERTAINITY - }, - 'stopId': stoptime.stop_id, - 'scheduleRelationship': 'SCHEDULED', - 'stop_time_properties': { - '[transit_realtime.stop_time_properties]': { - 'dropoffType': 'COORDINATE_WITH_DRIVER' if stoptime.drop_off_type == STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER else 'NONE', - 'pickupType': 'COORDINATE_WITH_DRIVER' if stoptime.pickup_type == STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER else 'NONE' - } - } - } - for stoptime in trip.stop_times] - - def _as_added_updates(self, trip, fromdate): - return [{ - 'trip': { - 'tripId': trip.trip_id, - 'startTime': trip.start_time_str(), - 'startDate': trip_date, - 'scheduleRelationship': 'ADDED', - 'routeId': trip.trip_id, - '[transit_realtime.trip_descriptor]': { - 'routeUrl' : trip.url, - 'agencyId' : trip.agency, - 'route_long_name' : trip.route_long_name(), - 'route_type': RIDESHARING_ROUTE_TYPE - } - }, - 'stopTimeUpdate': self._to_stop_times(trip, trip_date) - } for trip_date in trip.next_trip_dates(fromdate)] diff --git a/amarillo/services/gtfs_constants.py b/amarillo/services/gtfs_constants.py deleted file mode 100644 index 1e8f3af..0000000 --- a/amarillo/services/gtfs_constants.py +++ /dev/null @@ -1,14 +0,0 @@ -# Constants - -NO_BIKES_ALLOWED = 2 -RIDESHARING_ROUTE_TYPE = 1551 -CALENDAR_DATES_EXCEPTION_TYPE_ADDED = 1 -CALENDAR_DATES_EXCEPTION_TYPE_REMOVED = 2 -STOP_TIMES_STOP_TYPE_REGULARLY = 0 -STOP_TIMES_STOP_TYPE_NONE = 1 -STOP_TIMES_STOP_TYPE_PHONE_AGENCY = 2 -STOP_TIMES_STOP_TYPE_COORDINATE_DRIVER = 3 -STOP_TIMES_TIMEPOINT_APPROXIMATE = 0 -STOP_TIMES_TIMEPOINT_EXACT = 1 - -MFDZ_DEFAULT_UNCERTAINITY = 600 \ No newline at end of file diff --git a/amarillo/services/gtfsrt/__init__.py b/amarillo/services/gtfsrt/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/amarillo/services/gtfsrt/gtfs_realtime_pb2.py b/amarillo/services/gtfsrt/gtfs_realtime_pb2.py deleted file mode 100644 index 4e10463..0000000 --- a/amarillo/services/gtfsrt/gtfs_realtime_pb2.py +++ /dev/null @@ -1,80 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: gtfs-realtime.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13gtfs-realtime.proto\x12\x10transit_realtime\"y\n\x0b\x46\x65\x65\x64Message\x12,\n\x06header\x18\x01 \x02(\x0b\x32\x1c.transit_realtime.FeedHeader\x12,\n\x06\x65ntity\x18\x02 \x03(\x0b\x32\x1c.transit_realtime.FeedEntity*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xd7\x01\n\nFeedHeader\x12\x1d\n\x15gtfs_realtime_version\x18\x01 \x02(\t\x12Q\n\x0eincrementality\x18\x02 \x01(\x0e\x32+.transit_realtime.FeedHeader.Incrementality:\x0c\x46ULL_DATASET\x12\x11\n\ttimestamp\x18\x03 \x01(\x04\"4\n\x0eIncrementality\x12\x10\n\x0c\x46ULL_DATASET\x10\x00\x12\x10\n\x0c\x44IFFERENTIAL\x10\x01*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xd2\x01\n\nFeedEntity\x12\n\n\x02id\x18\x01 \x02(\t\x12\x19\n\nis_deleted\x18\x02 \x01(\x08:\x05\x66\x61lse\x12\x31\n\x0btrip_update\x18\x03 \x01(\x0b\x32\x1c.transit_realtime.TripUpdate\x12\x32\n\x07vehicle\x18\x04 \x01(\x0b\x32!.transit_realtime.VehiclePosition\x12&\n\x05\x61lert\x18\x05 \x01(\x0b\x32\x17.transit_realtime.Alert*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\x82\x08\n\nTripUpdate\x12.\n\x04trip\x18\x01 \x02(\x0b\x32 .transit_realtime.TripDescriptor\x12\x34\n\x07vehicle\x18\x03 \x01(\x0b\x32#.transit_realtime.VehicleDescriptor\x12\x45\n\x10stop_time_update\x18\x02 \x03(\x0b\x32+.transit_realtime.TripUpdate.StopTimeUpdate\x12\x11\n\ttimestamp\x18\x04 \x01(\x04\x12\r\n\x05\x64\x65lay\x18\x05 \x01(\x05\x12\x44\n\x0ftrip_properties\x18\x06 \x01(\x0b\x32+.transit_realtime.TripUpdate.TripProperties\x1aQ\n\rStopTimeEvent\x12\r\n\x05\x64\x65lay\x18\x01 \x01(\x05\x12\x0c\n\x04time\x18\x02 \x01(\x03\x12\x13\n\x0buncertainty\x18\x03 \x01(\x05*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\x1a\xa0\x04\n\x0eStopTimeUpdate\x12\x15\n\rstop_sequence\x18\x01 \x01(\r\x12\x0f\n\x07stop_id\x18\x04 \x01(\t\x12;\n\x07\x61rrival\x18\x02 \x01(\x0b\x32*.transit_realtime.TripUpdate.StopTimeEvent\x12=\n\tdeparture\x18\x03 \x01(\x0b\x32*.transit_realtime.TripUpdate.StopTimeEvent\x12j\n\x15schedule_relationship\x18\x05 \x01(\x0e\x32@.transit_realtime.TripUpdate.StopTimeUpdate.ScheduleRelationship:\tSCHEDULED\x12\\\n\x14stop_time_properties\x18\x06 \x01(\x0b\x32>.transit_realtime.TripUpdate.StopTimeUpdate.StopTimeProperties\x1a>\n\x12StopTimeProperties\x12\x18\n\x10\x61ssigned_stop_id\x18\x01 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"P\n\x14ScheduleRelationship\x12\r\n\tSCHEDULED\x10\x00\x12\x0b\n\x07SKIPPED\x10\x01\x12\x0b\n\x07NO_DATA\x10\x02\x12\x0f\n\x0bUNSCHEDULED\x10\x03*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\x1aY\n\x0eTripProperties\x12\x0f\n\x07trip_id\x18\x01 \x01(\t\x12\x12\n\nstart_date\x18\x02 \x01(\t\x12\x12\n\nstart_time\x18\x03 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xdf\t\n\x0fVehiclePosition\x12.\n\x04trip\x18\x01 \x01(\x0b\x32 .transit_realtime.TripDescriptor\x12\x34\n\x07vehicle\x18\x08 \x01(\x0b\x32#.transit_realtime.VehicleDescriptor\x12,\n\x08position\x18\x02 \x01(\x0b\x32\x1a.transit_realtime.Position\x12\x1d\n\x15\x63urrent_stop_sequence\x18\x03 \x01(\r\x12\x0f\n\x07stop_id\x18\x07 \x01(\t\x12Z\n\x0e\x63urrent_status\x18\x04 \x01(\x0e\x32\x33.transit_realtime.VehiclePosition.VehicleStopStatus:\rIN_TRANSIT_TO\x12\x11\n\ttimestamp\x18\x05 \x01(\x04\x12K\n\x10\x63ongestion_level\x18\x06 \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.CongestionLevel\x12K\n\x10occupancy_status\x18\t \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.OccupancyStatus\x12\x1c\n\x14occupancy_percentage\x18\n \x01(\r\x12Q\n\x16multi_carriage_details\x18\x0b \x03(\x0b\x32\x31.transit_realtime.VehiclePosition.CarriageDetails\x1a\xd9\x01\n\x0f\x43\x61rriageDetails\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05label\x18\x02 \x01(\t\x12^\n\x10occupancy_status\x18\x03 \x01(\x0e\x32\x31.transit_realtime.VehiclePosition.OccupancyStatus:\x11NO_DATA_AVAILABLE\x12 \n\x14occupancy_percentage\x18\x04 \x01(\x05:\x02-1\x12\x19\n\x11\x63\x61rriage_sequence\x18\x05 \x01(\r*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"G\n\x11VehicleStopStatus\x12\x0f\n\x0bINCOMING_AT\x10\x00\x12\x0e\n\nSTOPPED_AT\x10\x01\x12\x11\n\rIN_TRANSIT_TO\x10\x02\"}\n\x0f\x43ongestionLevel\x12\x1c\n\x18UNKNOWN_CONGESTION_LEVEL\x10\x00\x12\x14\n\x10RUNNING_SMOOTHLY\x10\x01\x12\x0f\n\x0bSTOP_AND_GO\x10\x02\x12\x0e\n\nCONGESTION\x10\x03\x12\x15\n\x11SEVERE_CONGESTION\x10\x04\"\xd9\x01\n\x0fOccupancyStatus\x12\t\n\x05\x45MPTY\x10\x00\x12\x18\n\x14MANY_SEATS_AVAILABLE\x10\x01\x12\x17\n\x13\x46\x45W_SEATS_AVAILABLE\x10\x02\x12\x16\n\x12STANDING_ROOM_ONLY\x10\x03\x12\x1e\n\x1a\x43RUSHED_STANDING_ROOM_ONLY\x10\x04\x12\x08\n\x04\x46ULL\x10\x05\x12\x1c\n\x18NOT_ACCEPTING_PASSENGERS\x10\x06\x12\x15\n\x11NO_DATA_AVAILABLE\x10\x07\x12\x11\n\rNOT_BOARDABLE\x10\x08*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\x80\t\n\x05\x41lert\x12\x32\n\ractive_period\x18\x01 \x03(\x0b\x32\x1b.transit_realtime.TimeRange\x12\x39\n\x0finformed_entity\x18\x05 \x03(\x0b\x32 .transit_realtime.EntitySelector\x12;\n\x05\x63\x61use\x18\x06 \x01(\x0e\x32\x1d.transit_realtime.Alert.Cause:\rUNKNOWN_CAUSE\x12>\n\x06\x65\x66\x66\x65\x63t\x18\x07 \x01(\x0e\x32\x1e.transit_realtime.Alert.Effect:\x0eUNKNOWN_EFFECT\x12/\n\x03url\x18\x08 \x01(\x0b\x32\".transit_realtime.TranslatedString\x12\x37\n\x0bheader_text\x18\n \x01(\x0b\x32\".transit_realtime.TranslatedString\x12<\n\x10\x64\x65scription_text\x18\x0b \x01(\x0b\x32\".transit_realtime.TranslatedString\x12;\n\x0ftts_header_text\x18\x0c \x01(\x0b\x32\".transit_realtime.TranslatedString\x12@\n\x14tts_description_text\x18\r \x01(\x0b\x32\".transit_realtime.TranslatedString\x12O\n\x0eseverity_level\x18\x0e \x01(\x0e\x32%.transit_realtime.Alert.SeverityLevel:\x10UNKNOWN_SEVERITY\"\xd8\x01\n\x05\x43\x61use\x12\x11\n\rUNKNOWN_CAUSE\x10\x01\x12\x0f\n\x0bOTHER_CAUSE\x10\x02\x12\x15\n\x11TECHNICAL_PROBLEM\x10\x03\x12\n\n\x06STRIKE\x10\x04\x12\x11\n\rDEMONSTRATION\x10\x05\x12\x0c\n\x08\x41\x43\x43IDENT\x10\x06\x12\x0b\n\x07HOLIDAY\x10\x07\x12\x0b\n\x07WEATHER\x10\x08\x12\x0f\n\x0bMAINTENANCE\x10\t\x12\x10\n\x0c\x43ONSTRUCTION\x10\n\x12\x13\n\x0fPOLICE_ACTIVITY\x10\x0b\x12\x15\n\x11MEDICAL_EMERGENCY\x10\x0c\"\xdd\x01\n\x06\x45\x66\x66\x65\x63t\x12\x0e\n\nNO_SERVICE\x10\x01\x12\x13\n\x0fREDUCED_SERVICE\x10\x02\x12\x16\n\x12SIGNIFICANT_DELAYS\x10\x03\x12\n\n\x06\x44\x45TOUR\x10\x04\x12\x16\n\x12\x41\x44\x44ITIONAL_SERVICE\x10\x05\x12\x14\n\x10MODIFIED_SERVICE\x10\x06\x12\x10\n\x0cOTHER_EFFECT\x10\x07\x12\x12\n\x0eUNKNOWN_EFFECT\x10\x08\x12\x0e\n\nSTOP_MOVED\x10\t\x12\r\n\tNO_EFFECT\x10\n\x12\x17\n\x13\x41\x43\x43\x45SSIBILITY_ISSUE\x10\x0b\"H\n\rSeverityLevel\x12\x14\n\x10UNKNOWN_SEVERITY\x10\x01\x12\x08\n\x04INFO\x10\x02\x12\x0b\n\x07WARNING\x10\x03\x12\n\n\x06SEVERE\x10\x04*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"7\n\tTimeRange\x12\r\n\x05start\x18\x01 \x01(\x04\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x04*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"q\n\x08Position\x12\x10\n\x08latitude\x18\x01 \x02(\x02\x12\x11\n\tlongitude\x18\x02 \x02(\x02\x12\x0f\n\x07\x62\x65\x61ring\x18\x03 \x01(\x02\x12\x10\n\x08odometer\x18\x04 \x01(\x01\x12\r\n\x05speed\x18\x05 \x01(\x02*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xcd\x02\n\x0eTripDescriptor\x12\x0f\n\x07trip_id\x18\x01 \x01(\t\x12\x10\n\x08route_id\x18\x05 \x01(\t\x12\x14\n\x0c\x64irection_id\x18\x06 \x01(\r\x12\x12\n\nstart_time\x18\x02 \x01(\t\x12\x12\n\nstart_date\x18\x03 \x01(\t\x12T\n\x15schedule_relationship\x18\x04 \x01(\x0e\x32\x35.transit_realtime.TripDescriptor.ScheduleRelationship\"t\n\x14ScheduleRelationship\x12\r\n\tSCHEDULED\x10\x00\x12\t\n\x05\x41\x44\x44\x45\x44\x10\x01\x12\x0f\n\x0bUNSCHEDULED\x10\x02\x12\x0c\n\x08\x43\x41NCELED\x10\x03\x12\x13\n\x0bREPLACEMENT\x10\x05\x1a\x02\x08\x01\x12\x0e\n\nDUPLICATED\x10\x06*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"U\n\x11VehicleDescriptor\x12\n\n\x02id\x18\x01 \x01(\t\x12\r\n\x05label\x18\x02 \x01(\t\x12\x15\n\rlicense_plate\x18\x03 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xb0\x01\n\x0e\x45ntitySelector\x12\x11\n\tagency_id\x18\x01 \x01(\t\x12\x10\n\x08route_id\x18\x02 \x01(\t\x12\x12\n\nroute_type\x18\x03 \x01(\x05\x12.\n\x04trip\x18\x04 \x01(\x0b\x32 .transit_realtime.TripDescriptor\x12\x0f\n\x07stop_id\x18\x05 \x01(\t\x12\x14\n\x0c\x64irection_id\x18\x06 \x01(\r*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N\"\xa6\x01\n\x10TranslatedString\x12\x43\n\x0btranslation\x18\x01 \x03(\x0b\x32..transit_realtime.TranslatedString.Translation\x1a=\n\x0bTranslation\x12\x0c\n\x04text\x18\x01 \x02(\t\x12\x10\n\x08language\x18\x02 \x01(\t*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90N*\x06\x08\xe8\x07\x10\xd0\x0f*\x06\x08\xa8\x46\x10\x90NB\x1d\n\x1b\x63om.google.transit.realtime') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'gtfs_realtime_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\033com.google.transit.realtime' - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP.values_by_name["REPLACEMENT"]._options = None - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP.values_by_name["REPLACEMENT"]._serialized_options = b'\010\001' - _FEEDMESSAGE._serialized_start=41 - _FEEDMESSAGE._serialized_end=162 - _FEEDHEADER._serialized_start=165 - _FEEDHEADER._serialized_end=380 - _FEEDHEADER_INCREMENTALITY._serialized_start=312 - _FEEDHEADER_INCREMENTALITY._serialized_end=364 - _FEEDENTITY._serialized_start=383 - _FEEDENTITY._serialized_end=593 - _TRIPUPDATE._serialized_start=596 - _TRIPUPDATE._serialized_end=1622 - _TRIPUPDATE_STOPTIMEEVENT._serialized_start=887 - _TRIPUPDATE_STOPTIMEEVENT._serialized_end=968 - _TRIPUPDATE_STOPTIMEUPDATE._serialized_start=971 - _TRIPUPDATE_STOPTIMEUPDATE._serialized_end=1515 - _TRIPUPDATE_STOPTIMEUPDATE_STOPTIMEPROPERTIES._serialized_start=1355 - _TRIPUPDATE_STOPTIMEUPDATE_STOPTIMEPROPERTIES._serialized_end=1417 - _TRIPUPDATE_STOPTIMEUPDATE_SCHEDULERELATIONSHIP._serialized_start=1419 - _TRIPUPDATE_STOPTIMEUPDATE_SCHEDULERELATIONSHIP._serialized_end=1499 - _TRIPUPDATE_TRIPPROPERTIES._serialized_start=1517 - _TRIPUPDATE_TRIPPROPERTIES._serialized_end=1606 - _VEHICLEPOSITION._serialized_start=1625 - _VEHICLEPOSITION._serialized_end=2872 - _VEHICLEPOSITION_CARRIAGEDETAILS._serialized_start=2219 - _VEHICLEPOSITION_CARRIAGEDETAILS._serialized_end=2436 - _VEHICLEPOSITION_VEHICLESTOPSTATUS._serialized_start=2438 - _VEHICLEPOSITION_VEHICLESTOPSTATUS._serialized_end=2509 - _VEHICLEPOSITION_CONGESTIONLEVEL._serialized_start=2511 - _VEHICLEPOSITION_CONGESTIONLEVEL._serialized_end=2636 - _VEHICLEPOSITION_OCCUPANCYSTATUS._serialized_start=2639 - _VEHICLEPOSITION_OCCUPANCYSTATUS._serialized_end=2856 - _ALERT._serialized_start=2875 - _ALERT._serialized_end=4027 - _ALERT_CAUSE._serialized_start=3497 - _ALERT_CAUSE._serialized_end=3713 - _ALERT_EFFECT._serialized_start=3716 - _ALERT_EFFECT._serialized_end=3937 - _ALERT_SEVERITYLEVEL._serialized_start=3939 - _ALERT_SEVERITYLEVEL._serialized_end=4011 - _TIMERANGE._serialized_start=4029 - _TIMERANGE._serialized_end=4084 - _POSITION._serialized_start=4086 - _POSITION._serialized_end=4199 - _TRIPDESCRIPTOR._serialized_start=4202 - _TRIPDESCRIPTOR._serialized_end=4535 - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP._serialized_start=4403 - _TRIPDESCRIPTOR_SCHEDULERELATIONSHIP._serialized_end=4519 - _VEHICLEDESCRIPTOR._serialized_start=4537 - _VEHICLEDESCRIPTOR._serialized_end=4622 - _ENTITYSELECTOR._serialized_start=4625 - _ENTITYSELECTOR._serialized_end=4801 - _TRANSLATEDSTRING._serialized_start=4804 - _TRANSLATEDSTRING._serialized_end=4970 - _TRANSLATEDSTRING_TRANSLATION._serialized_start=4893 - _TRANSLATEDSTRING_TRANSLATION._serialized_end=4954 -# @@protoc_insertion_point(module_scope) diff --git a/amarillo/services/gtfsrt/realtime_extension_pb2.py b/amarillo/services/gtfsrt/realtime_extension_pb2.py deleted file mode 100644 index 5db1fda..0000000 --- a/amarillo/services/gtfsrt/realtime_extension_pb2.py +++ /dev/null @@ -1,33 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: realtime_extension.proto -"""Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -import amarillo.services.gtfsrt.gtfs_realtime_pb2 as gtfs__realtime__pb2 - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18realtime_extension.proto\x12\x10transit_realtime\x1a\x13gtfs-realtime.proto\"p\n\x1bMfdzTripDescriptorExtension\x12\x11\n\troute_url\x18\x01 \x01(\t\x12\x11\n\tagency_id\x18\x02 \x01(\t\x12\x17\n\x0froute_long_name\x18\x03 \x01(\t\x12\x12\n\nroute_type\x18\x04 \x01(\r\"\xb0\x02\n\x1fMfdzStopTimePropertiesExtension\x12X\n\x0bpickup_type\x18\x01 \x01(\x0e\x32\x43.transit_realtime.MfdzStopTimePropertiesExtension.DropOffPickupType\x12Y\n\x0c\x64ropoff_type\x18\x02 \x01(\x0e\x32\x43.transit_realtime.MfdzStopTimePropertiesExtension.DropOffPickupType\"X\n\x11\x44ropOffPickupType\x12\x0b\n\x07REGULAR\x10\x00\x12\x08\n\x04NONE\x10\x01\x12\x10\n\x0cPHONE_AGENCY\x10\x02\x12\x1a\n\x16\x43OORDINATE_WITH_DRIVER\x10\x03:i\n\x0ftrip_descriptor\x12 .transit_realtime.TripDescriptor\x18\xf5\x07 \x01(\x0b\x32-.transit_realtime.MfdzTripDescriptorExtension:\x90\x01\n\x14stop_time_properties\x12>.transit_realtime.TripUpdate.StopTimeUpdate.StopTimeProperties\x18\xf5\x07 \x01(\x0b\x32\x31.transit_realtime.MfdzStopTimePropertiesExtensionB\t\n\x07\x64\x65.mfdz') - -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'realtime_extension_pb2', globals()) -if _descriptor._USE_C_DESCRIPTORS == False: - gtfs__realtime__pb2.TripDescriptor.RegisterExtension(trip_descriptor) - gtfs__realtime__pb2.TripUpdate.StopTimeUpdate.StopTimeProperties.RegisterExtension(stop_time_properties) - - DESCRIPTOR._options = None - DESCRIPTOR._serialized_options = b'\n\007de.mfdz' - _MFDZTRIPDESCRIPTOREXTENSION._serialized_start=67 - _MFDZTRIPDESCRIPTOREXTENSION._serialized_end=179 - _MFDZSTOPTIMEPROPERTIESEXTENSION._serialized_start=182 - _MFDZSTOPTIMEPROPERTIESEXTENSION._serialized_end=486 - _MFDZSTOPTIMEPROPERTIESEXTENSION_DROPOFFPICKUPTYPE._serialized_start=398 - _MFDZSTOPTIMEPROPERTIESEXTENSION_DROPOFFPICKUPTYPE._serialized_end=486 -# @@protoc_insertion_point(module_scope) From 48f6fd97e60b33c0dfd292b204cd53f8b9c7ada6 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 8 Feb 2024 11:45:43 +0100 Subject: [PATCH 15/50] moved gtfs model to enhancer --- amarillo/models/gtfs.py | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 amarillo/models/gtfs.py diff --git a/amarillo/models/gtfs.py b/amarillo/models/gtfs.py deleted file mode 100644 index ee7f701..0000000 --- a/amarillo/models/gtfs.py +++ /dev/null @@ -1,29 +0,0 @@ -from collections import namedtuple -from datetime import timedelta - -GtfsFeedInfo = namedtuple('GtfsFeedInfo', 'feed_id feed_publisher_name feed_publisher_url feed_lang feed_version') -GtfsAgency = namedtuple('GtfsAgency', 'agency_id agency_name agency_url agency_timezone agency_lang agency_email') -GtfsRoute = namedtuple('GtfsRoute', 'agency_id route_id route_long_name route_type route_url route_short_name') -GtfsStop = namedtuple('GtfsStop', 'stop_id stop_lat stop_lon stop_name') -GtfsStopTime = namedtuple('GtfsStopTime', 'trip_id departure_time arrival_time stop_id stop_sequence pickup_type drop_off_type timepoint') -GtfsTrip = namedtuple('GtfsTrip', 'route_id trip_id service_id shape_id trip_headsign bikes_allowed') -GtfsCalendar = namedtuple('GtfsCalendar', 'service_id start_date end_date monday tuesday wednesday thursday friday saturday sunday') -GtfsCalendarDate = namedtuple('GtfsCalendarDate', 'service_id date exception_type') -GtfsShape = namedtuple('GtfsShape','shape_id shape_pt_lon shape_pt_lat shape_pt_sequence') - -# TODO Move to utils -class GtfsTimeDelta(timedelta): - def __str__(self): - seconds = self.total_seconds() - hours = seconds // 3600 - minutes = (seconds % 3600) // 60 - seconds = seconds % 60 - str = '{:02d}:{:02d}:{:02d}'.format(int(hours), int(minutes), int(seconds)) - return (str) - - def __add__(self, other): - if isinstance(other, timedelta): - return self.__class__(self.days + other.days, - self.seconds + other.seconds, - self.microseconds + other.microseconds) - return NotImplemented \ No newline at end of file From b702756d66c7836d50d50f0057d4eda0edb50aa2 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Fri, 9 Feb 2024 13:54:32 +0100 Subject: [PATCH 16/50] moved region/gtfs endpoint to export plugin --- amarillo/routers/region.py | 33 +-------------------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/amarillo/routers/region.py b/amarillo/routers/region.py index 50da511..a530522 100644 --- a/amarillo/routers/region.py +++ b/amarillo/routers/region.py @@ -45,7 +45,7 @@ async def get_region(region_id: str) -> Region: return region def _assert_region_exists(region_id: str) -> Region: - regions: regionService = container['regions'] + regions: RegionService = container['regions'] region = regions.get_region(region_id) region_exists = region is not None @@ -55,34 +55,3 @@ def _assert_region_exists(region_id: str) -> Region: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=message) return region - -@router.get("/{region_id}/gtfs", - summary="Return GTFS Feed for this region", - response_description="GTFS-Feed (zip-file)", - response_class=FileResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"description": "Region not found"}, - } - ) -async def get_file(region_id: str, user: str = Depends(verify_admin_api_key)): - _assert_region_exists(region_id) - return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfs.zip') - -@router.get("/{region_id}/gtfs-rt", - summary="Return GTFS-RT Feed for this region", - response_description="GTFS-RT-Feed", - response_class=FileResponse, - responses={ - status.HTTP_404_NOT_FOUND: {"description": "Region not found"}, - status.HTTP_400_BAD_REQUEST: {"description": "Bad request, e.g. because format is not supported, i.e. neither protobuf nor json."} - } - ) -async def get_file(region_id: str, format: str = 'protobuf', user: str = Depends(verify_admin_api_key)): - _assert_region_exists(region_id) - if format == 'json': - return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfsrt.json') - elif format == 'protobuf': - return FileResponse(f'data/gtfs/amarillo.{region_id}.gtfsrt.pbf') - else: - message = "Specified format is not supported, i.e. neither protobuf nor json." - raise HTTPException(status_code=status.HTTP_400_BAD_REQUEST, detail=message) From 3063e1a98396f151765ec7fba2474c364bb31c60 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Fri, 9 Feb 2024 13:54:48 +0100 Subject: [PATCH 17/50] carpool GRFS attributes --- amarillo/models/Carpool.py | 110 +++++++++++++++++++++++++++++++++++- amarillo/routers/agency.py | 1 + amarillo/routers/carpool.py | 2 + 3 files changed, 112 insertions(+), 1 deletion(-) diff --git a/amarillo/models/Carpool.py b/amarillo/models/Carpool.py index ffbb995..5ee5bdc 100644 --- a/amarillo/models/Carpool.py +++ b/amarillo/models/Carpool.py @@ -4,7 +4,7 @@ from datetime import time from pydantic import BaseModel, Field from geojson_pydantic.geometries import LineString -from enum import Enum +from enum import Enum, IntEnum NumType = Union[float, int] @@ -24,6 +24,15 @@ class PickupDropoffType(str, Enum): only_pickup = "only_pickup" only_dropoff = "only_dropoff" +class YesNoEnum(IntEnum): + yes = 1 + no = 2 + +class LuggageSize(IntEnum): + small = 1 + medium = 2 + large = 3 + class StopTime(BaseModel): id: Optional[str] = Field( None, @@ -111,7 +120,83 @@ class Region(BaseModel): bbox: Tuple[NumType, NumType, NumType, NumType] = Field( description="Bounding box of the region. Format is [minLon, minLat, maxLon, maxLat]", examples=[[10.5,49.2,11.3,51.3]]) + +class RidesharingInfo(BaseModel): + number_free_seats: int = Field( + description="Number of free seats", + ge=0, + examples=[3]) + + same_gender: Optional[YesNoEnum] = Field( + None, + description="Trip only for same gender:" + "1: Yes" + "2: No", + examples=[1]) + luggage_size: Optional[LuggageSize] = Field( + None, + description="Size of the luggage:" + "1: small" + "2: medium" + "3: large", + examples=[3]) + animal_car: Optional[YesNoEnum] = Field( + None, + description="Animals in Car allowed:" + "1: Yes" + "2: No", + examples=[2]) + car_model: Optional[str] = Field( + None, + description="Car model", + min_length=1, + max_length=48, + examples=["Golf"]) + car_brand: Optional[str] = Field( + None, + description="Car brand", + min_length=1, + max_length=48, + examples=["VW"]) + + creation_date: datetime = Field( + description="Date when trip was created", + examples=["2022-02-13T20:20:39+00:00"]) + + smoking: Optional[YesNoEnum] = Field( + None, + description="Smoking allowed:" + "1: Yes" + "2: No", + examples=[2]) + + payment_method: Optional[str] = Field( + None, + description="Method of payment", + min_length=1, + max_length=48) + +class Driver(BaseModel): + driver_id: Optional[str] = Field( + None, + description="Identifies the driver.", + min_length=1, + max_length=256, + pattern='^[a-zA-Z0-9_-]+$', + examples=["789"]) + profile_picture: Optional[HttpUrl] = Field( + None, + description="URL that contains the profile picture", + examples=["https://mfdz.de/driver/789/picture"]) + rating: Optional[int] = Field( + None, + description="Rating of the driver from 1 to 5." + "0 no rating yet", + ge=0, + le=5, + examples=[5]) + class Agency(BaseModel): id: str = Field( description="ID of the agency.", @@ -196,6 +281,17 @@ class Carpool(BaseModel): max_length=20, pattern='^[a-zA-Z0-9]+$', examples=["mfdz"]) + + driver: Optional[Driver] = Field( + None, + description="Driver data", + examples=[""" + { + "driver_id": "123", + "profile_picture": "https://mfdz.de/driver/789/picture", + "rating": 5 + } + """]) deeplink: HttpUrl = Field( description="Link to an information page providing detail information " @@ -258,6 +354,18 @@ class Carpool(BaseModel): "purge outdated offers (e.g. older than 180 days). If not " "passed, the service may assume 'now'", examples=["2022-02-13T20:20:39+00:00"]) + additional_ridesharing_info: Optional[RidesharingInfo] = Field( + None, + description="Extension of GRFS to the GTFS standard", + examples=[""" + { + "number_free_seats": 2, + "creation_date": "2022-02-13T20:20:39+00:00", + "same_gender": 2, + "smoking": 1, + "luggage_size": 3 + } + """]) model_config = ConfigDict(json_schema_extra={ "title": "Carpool", # description ... diff --git a/amarillo/routers/agency.py b/amarillo/routers/agency.py index cf50c47..5b09db4 100644 --- a/amarillo/routers/agency.py +++ b/amarillo/routers/agency.py @@ -52,6 +52,7 @@ async def get_agency(agency_id: str, admin_api_key: str = Depends(verify_api_key operation_id="sync", summary="Synchronizes all carpool offers", response_model=List[Carpool], + response_model_exclude_none=True, responses={ status.HTTP_200_OK: { "description": "Carpool created"}, diff --git a/amarillo/routers/carpool.py b/amarillo/routers/carpool.py index 1eeb96a..0003fcc 100644 --- a/amarillo/routers/carpool.py +++ b/amarillo/routers/carpool.py @@ -25,6 +25,7 @@ summary="Add a new or update existing carpool", description="Carpool object to be created or updated", response_model=Carpool, + response_model_exclude_none=True, responses={ status.HTTP_404_NOT_FOUND: { "description": "Agency does not exist"}, @@ -48,6 +49,7 @@ async def post_carpool(carpool: Carpool = Body(..., examples=examples), operation_id="getcarpoolById", summary="Find carpool by ID", response_model=Carpool, + response_model_exclude_none=True, description="Find carpool by ID", responses={ status.HTTP_404_NOT_FOUND: {"description": "Carpool not found"}, From ea7a2d6d2481ddc2b9323a78f42d1d78aab72d3b Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 13 May 2024 15:33:07 +0200 Subject: [PATCH 18/50] __init__.py EOL, extra secrets clarification --- amarillo/plugins/__init__.py | 2 +- amarillo/services/secrets.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/amarillo/plugins/__init__.py b/amarillo/plugins/__init__.py index 0260537..69e3be5 100644 --- a/amarillo/plugins/__init__.py +++ b/amarillo/plugins/__init__.py @@ -1 +1 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) \ No newline at end of file +__path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/amarillo/services/secrets.py b/amarillo/services/secrets.py index 756d21c..773039e 100644 --- a/amarillo/services/secrets.py +++ b/amarillo/services/secrets.py @@ -2,7 +2,7 @@ from pydantic_settings import BaseSettings # Example: secrets = { "mfdz": "some secret" } class Secrets(BaseSettings): - model_config = ConfigDict(extra='allow') + model_config = ConfigDict(extra='allow') # Allow plugins to add extra values ride2go_token: str = Field(None, env = 'RIDE2GO_TOKEN') From 456a1902e2865ea44ce8b63afa0b42dac3195ab7 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 19 Feb 2024 11:13:23 +0100 Subject: [PATCH 19/50] CRUD metrics --- amarillo/routers/carpool.py | 24 +++++++++++++++++++++--- amarillo/services/carpools.py | 5 +++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/amarillo/routers/carpool.py b/amarillo/routers/carpool.py index 0003fcc..f793578 100644 --- a/amarillo/routers/carpool.py +++ b/amarillo/routers/carpool.py @@ -38,9 +38,7 @@ async def post_carpool(carpool: Carpool = Body(..., examples=examples), logger.info(f"POST trip {carpool.agency}:{carpool.id}.") await assert_agency_exists(carpool.agency) - await set_lastUpdated_if_unset(carpool) - - await save_carpool(carpool) + await store_carpool(carpool) return carpool @@ -92,10 +90,30 @@ async def _delete_carpool(agency_id: str, carpool_id: str): logger.info(f"Saved carpool {agency_id}:{carpool_id} in trash.") os.remove(f"data/carpool/{agency_id}/{carpool_id}.json") + try: + from amarillo.plugins.metrics import trips_deleted_counter + trips_deleted_counter.inc() + except ImportError: + pass + + async def store_carpool(carpool: Carpool) -> Carpool: + carpool_exists = os.path.exists(f"data/carpool/{carpool.agency}/{carpool.id}.json") + await set_lastUpdated_if_unset(carpool) await save_carpool(carpool) + try: + from amarillo.plugins.metrics import trips_created_counter, trips_updated_counter + if(carpool_exists): + # logger.info("Incrementing trips updated") + trips_updated_counter.inc() + else: + # logger.info("Incrementing trips created") + trips_created_counter.inc() + except ImportError: + pass + return carpool async def set_lastUpdated_if_unset(carpool): diff --git a/amarillo/services/carpools.py b/amarillo/services/carpools.py index 7790cbc..7b6fd39 100644 --- a/amarillo/services/carpools.py +++ b/amarillo/services/carpools.py @@ -37,6 +37,11 @@ def purge_outdated_offers(self): if cp and self.is_outdated(cp): logger.info("Purge outdated offer %s", key) self.delete(cp.agency, cp.id) + try: + from amarillo.plugins.metrics import trips_deleted_counter + trips_deleted_counter.inc() + except ImportError: + pass def get(self, agency_id: str, carpool_id: str): return self.carpools.get(f"{agency_id}:{carpool_id}") From 76ffd337a873ce5ebcacf40d0f33479d820d5c78 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Wed, 21 Feb 2024 12:39:06 +0100 Subject: [PATCH 20/50] Added route_color and route_text_color to Carpool model --- amarillo/models/Carpool.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/amarillo/models/Carpool.py b/amarillo/models/Carpool.py index 5ee5bdc..b9c019c 100644 --- a/amarillo/models/Carpool.py +++ b/amarillo/models/Carpool.py @@ -342,7 +342,22 @@ class Carpool(BaseModel): "published.", examples=['A single date 2022-04-04 or a list of weekdays ["saturday", ' '"sunday"]']) - + route_color: Optional[str] = Field( + None, + pattern='^([0-9A-Fa-f]{6})$', + description="Route color designation that matches public facing material. " + "The color difference between route_color and route_text_color " + "should provide sufficient contrast when viewed on a black and " + "white screen.", + examples=["0039A6"]) + route_text_color: Optional[str] = Field( + None, + pattern='^([0-9A-Fa-f]{6})$', + description="Legible color to use for text drawn against a background of " + "route_color. The color difference between route_color and " + "route_text_color should provide sufficient contrast when " + "viewed on a black and white screen.", + examples=["D4D2D2"]) path: Optional[LineString] = Field( None, description="Optional route geometry as json LineString.") From 89953091300112f6ee7e206957c8c06974920882 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 1 Aug 2024 13:11:44 +0200 Subject: [PATCH 21/50] Allow plugins to add extra configuration values --- amarillo/services/config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/amarillo/services/config.py b/amarillo/services/config.py index 1dd620a..e55eec7 100644 --- a/amarillo/services/config.py +++ b/amarillo/services/config.py @@ -1,4 +1,5 @@ from typing import List +from pydantic import ConfigDict from pydantic_settings import BaseSettings @@ -10,4 +11,6 @@ class Config(BaseSettings): graphhopper_base_url: str = 'https://api.mfdz.de/gh' stop_sources_file: str = 'conf/stop_sources.json' + model_config = ConfigDict(extra='allow') # Allow plugins to add extra values + config = Config(_env_file='config', _env_file_encoding='utf-8') From 4af9cf3e0be05a5a31bdb882b3e9dc3a8a33c450 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 1 Aug 2024 13:14:26 +0200 Subject: [PATCH 22/50] Moved stop importer tests to amarillo-stops --- amarillo/services/stop_importer/__init__.py | 8 -- amarillo/services/stop_importer/base.py | 79 ------------------ amarillo/services/stop_importer/gtfs.py | 29 ------- amarillo/services/stop_importer/overpass.py | 39 --------- amarillo/tests/fixtures/stops.gtfs.zip | Bin 777 -> 0 bytes .../tests/fixtures/stops_overpass_result.csv | 40 --------- amarillo/tests/test_stops_gtfs_importer.py | 15 ---- .../tests/test_stops_overpass_importer.py | 14 ---- 8 files changed, 224 deletions(-) delete mode 100644 amarillo/services/stop_importer/__init__.py delete mode 100644 amarillo/services/stop_importer/base.py delete mode 100644 amarillo/services/stop_importer/gtfs.py delete mode 100644 amarillo/services/stop_importer/overpass.py delete mode 100644 amarillo/tests/fixtures/stops.gtfs.zip delete mode 100644 amarillo/tests/fixtures/stops_overpass_result.csv delete mode 100644 amarillo/tests/test_stops_gtfs_importer.py delete mode 100644 amarillo/tests/test_stops_overpass_importer.py diff --git a/amarillo/services/stop_importer/__init__.py b/amarillo/services/stop_importer/__init__.py deleted file mode 100644 index 0c9769a..0000000 --- a/amarillo/services/stop_importer/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -from .base import ( - CsvStopsImporter as CsvStopsImporter, -) -from .base import ( - GeojsonStopsImporter as GeojsonStopsImporter, -) -from .gtfs import GtfsStopsImporter as GtfsStopsImporter -from .overpass import OverpassStopsImporter as OverpassStopsImporter diff --git a/amarillo/services/stop_importer/base.py b/amarillo/services/stop_importer/base.py deleted file mode 100644 index b80c99b..0000000 --- a/amarillo/services/stop_importer/base.py +++ /dev/null @@ -1,79 +0,0 @@ -import codecs -import csv -import logging -import re - -import geopandas as gpd -import requests - -logger = logging.getLogger(__name__) - - -class BaseStopsImporter: - def _normalize_stop_name(self, stop_name): - # if the name is empty, we set P+R as a fall back. However, it should be named at the source - default_name = 'P+R' - if stop_name in ('', 'Park&Ride'): - return default_name - return re.sub(r'P(ark)?\s?[\+&]\s?R(ail|ide)?', 'P+R', stop_name) - - def _as_dataframe(self, id, lat, lon, stop_name): - df = gpd.GeoDataFrame(data={'x': lon, 'y': lat, 'stop_name': stop_name, 'id': id}) - return gpd.GeoDataFrame(df, geometry=gpd.points_from_xy(df.x, df.y, crs='EPSG:4326')) - - -class CsvStopsImporter(BaseStopsImporter): - DEFAULT_COLUMN_MAPPING = { - 'stop_id': 'stop_id', - 'stop_lat': 'stop_lat', - 'stop_lon': 'stop_lon', - 'stop_name': 'stop_name', - } - - def load_stops(self, source, timeout=15): - if source.startswith('http'): - with requests.get(source, timeout=timeout) as csv_source: - return self._load_stops_from_csv_source( - codecs.iterdecode(csv_source.iter_lines(), 'utf-8'), delimiter=';' - ) - else: - with open(source, encoding='utf-8') as csv_source: - return self._load_stops_from_csv_source(csv_source, delimiter=';') - - def _load_stops_from_csv_source(self, csv_source, delimiter: str = ',', column_mapping=None): - if column_mapping is None: - column_mapping = self.DEFAULT_COLUMN_MAPPING - id = [] - lat = [] - lon = [] - stop_name = [] - reader = csv.DictReader(csv_source, delimiter=delimiter) - for row in reader: - id.append(row[column_mapping['stop_id']]) - lat.append(float(row[column_mapping['stop_lat']].replace(',', '.'))) - lon.append(float(row[column_mapping['stop_lon']].replace(',', '.'))) - stop_name.append(self._normalize_stop_name(row[column_mapping['stop_name']])) - - return self._as_dataframe(id, lat, lon, stop_name) - - -class GeojsonStopsImporter(BaseStopsImporter): - def load_stops(self, source, timeout=15): - with requests.get(source, timeout=timeout) as json_source: - geojson_source = json_source.json() - id = [] - lat = [] - lon = [] - stop_name = [] - for row in geojson_source['features']: - coord = row['geometry']['coordinates'] - if not coord or not row['properties'].get('name'): - logger.error('Stop feature {} has null coord or name'.format(row['id'])) - continue - - id.append(row['id']) - lon.append(coord[0]) - lat.append(coord[1]) - stop_name.append(self._normalize_stop_name(row['properties']['name'])) - - return self._as_dataframe(id, lat, lon, stop_name) diff --git a/amarillo/services/stop_importer/gtfs.py b/amarillo/services/stop_importer/gtfs.py deleted file mode 100644 index b895b5f..0000000 --- a/amarillo/services/stop_importer/gtfs.py +++ /dev/null @@ -1,29 +0,0 @@ -import io -import zipfile -from pathlib import Path - -import requests - -from .base import CsvStopsImporter - - -class GtfsStopsImporter(CsvStopsImporter): - def load_stops(self, id, url, timeout=15, **kwargs): - if url.startswith('http'): - # TODO: only reload if file is older than x - gtfs_file = Path(f'data/{id}.gtfs.zip') - with requests.get(url, timeout=timeout) as response: - if response.ok: - self._store_response(gtfs_file, response) - else: - gtfs_file = url - - with zipfile.ZipFile(gtfs_file) as gtfs: - with gtfs.open('stops.txt', 'r') as stops_file: - return self._load_stops_from_csv_source(io.TextIOWrapper(stops_file, 'utf-8-sig')) - - def _store_response(self, filename, response): - with filename.open('wb') as file: - for chunk in response.iter_content(chunk_size=1024 * 1024): - if chunk: - file.write(chunk) diff --git a/amarillo/services/stop_importer/overpass.py b/amarillo/services/stop_importer/overpass.py deleted file mode 100644 index 8067471..0000000 --- a/amarillo/services/stop_importer/overpass.py +++ /dev/null @@ -1,39 +0,0 @@ -import csv -import io -import logging - -import requests - -from .base import BaseStopsImporter - -logger = logging.getLogger(__name__) - - -class OverpassStopsImporter(BaseStopsImporter): - def load_stops(self, area_selector, timeout=15, **kwargs): - query = f''' - [out:csv(::"type", ::"id", ::"lat", ::"lon", name,parking,park_ride,operator,access,lit,fee,capacity,"capacity:disabled",supervised,surface,covered,maxstay,opening_hours)][timeout:60]; - area{area_selector}->.a; - nwr(area.a)[park_ride][park_ride!=no][access!=customers]; - out center; - ''' - - response = requests.post('https://overpass-api.de/api/interpreter', data=query, timeout=timeout) - if not response.ok: - logger.error(f'Error retrieving stops from overpass: {response.text}') - - return self._parse_overpass_csv_response(response.text.splitlines()) - - def _parse_overpass_csv_response(self, csv_source): - id = [] - lat = [] - lon = [] - stop_name = [] - reader = csv.DictReader(csv_source, delimiter='\t') - for row in reader: - id.append(f'osm:{row["@type"][0]}{row["@id"]}') - lat.append(float(row['@lat'])) - lon.append(float(row['@lon'])) - stop_name.append(self._normalize_stop_name(row['name'])) - - return self._as_dataframe(id, lat, lon, stop_name) diff --git a/amarillo/tests/fixtures/stops.gtfs.zip b/amarillo/tests/fixtures/stops.gtfs.zip deleted file mode 100644 index a1a2d891f2b42027b92d93a16b12042487bdbefc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 777 zcmWIWW@Zs#-~hrO9>oz1P~gqTz`)6%z))P0Ur?-9Qc)5b!pp#Z)%|BG43}1LGcdAz z1*!vTXJE)Z@11qnK%n7atwOm*+-!rYb0<#gc$r%^3vOMYcJ9Ott)<^z>h5}>#C}TD zsCTZbv*Ut?@ikJh`|6)fI==1wtJ`NkPMMb3kt%C+-MOxJ>G?a;&Kq7$o+C89OmF+P zyCJbz4OIY+T?-gf zgExkSX@6KTB|1Lp#@lCH0p9E!9xq`eZ6?> z!Sx6jY-quioFJN)L5)l$nvu7@wDWN7kE803; 0 - assert stopsDataFrames.loc[0, ['x', 'y', 'stop_name', 'id']].values.tolist() == [ - 8.75033716398694, - 48.7891850492262, - 'Monakam Brunnenstr.', - 'de:08235:4060:0:3', - ] diff --git a/amarillo/tests/test_stops_overpass_importer.py b/amarillo/tests/test_stops_overpass_importer.py deleted file mode 100644 index 7e6a4b4..0000000 --- a/amarillo/tests/test_stops_overpass_importer.py +++ /dev/null @@ -1,14 +0,0 @@ -from amarillo.services.stop_importer.overpass import OverpassStopsImporter - - -def test_load_geojson_stops_from_web_(): - with open('amarillo/tests/fixtures/stops_overpass_result.csv') as f: - stopsDataFrames = OverpassStopsImporter()._parse_overpass_csv_response(f) - - assert len(stopsDataFrames) > 0 - assert stopsDataFrames.loc[0, ['x', 'y', 'stop_name', 'id']].values.tolist() == [ - 11.7621206, - 46.7470278, - 'P+R', - 'osm:n5206558994', - ] From 7db9706139026394b129b357014e6849ec8adb4a Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Fri, 17 May 2024 12:58:02 +0200 Subject: [PATCH 23/50] Call enhancer in background task --- amarillo/routers/carpool.py | 28 +++++++++++++++++++++++++--- amarillo/services/config.py | 1 + 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/amarillo/routers/carpool.py b/amarillo/routers/carpool.py index f793578..c741aa9 100644 --- a/amarillo/routers/carpool.py +++ b/amarillo/routers/carpool.py @@ -5,13 +5,16 @@ import re from glob import glob -from fastapi import APIRouter, Body, Header, HTTPException, status, Depends +from fastapi import APIRouter, Body, Header, HTTPException, status, Depends, BackgroundTasks +import requests from datetime import datetime from amarillo.models.Carpool import Carpool from amarillo.routers.agencyconf import verify_api_key, verify_permission_for_same_agency_or_admin from amarillo.tests.sampledata import examples +from amarillo.services.config import config +from amarillo.utils.utils import assert_folder_exists logger = logging.getLogger(__name__) @@ -20,6 +23,19 @@ tags=["carpool"] ) +#TODO: housekeeping for outdated trips + +def enhance_trip(carpool: Carpool): + response = requests.post(f"{config.enhancer_url}", carpool.model_dump_json()) + enhanced_carpool = Carpool(**json.loads(response.content)) + + folder = f'data/enhanced/{carpool.agency}' + filename = f'{folder}/{carpool.id}.json' + + assert_folder_exists(folder) + with open(filename, 'w', encoding='utf-8') as f: + f.write(enhanced_carpool.model_dump_json()) + @router.post("/", operation_id="addcarpool", summary="Add a new or update existing carpool", @@ -31,7 +47,7 @@ "description": "Agency does not exist"}, }) -async def post_carpool(carpool: Carpool = Body(..., examples=examples), +async def post_carpool(background_tasks: BackgroundTasks, carpool: Carpool = Body(..., examples=examples), requesting_agency_id: str = Depends(verify_api_key)) -> Carpool: await verify_permission_for_same_agency_or_admin(carpool.agency, requesting_agency_id) @@ -40,6 +56,8 @@ async def post_carpool(carpool: Carpool = Body(..., examples=examples), await store_carpool(carpool) + background_tasks.add_task(enhance_trip, carpool) + return carpool # TODO 403 @@ -88,7 +106,11 @@ async def _delete_carpool(agency_id: str, carpool_id: str): # load and store, to receive pyinotify events and have file timestamp updated await save_carpool(cp, 'data/trash') logger.info(f"Saved carpool {agency_id}:{carpool_id} in trash.") - os.remove(f"data/carpool/{agency_id}/{carpool_id}.json") + try: + os.remove(f"data/carpool/{agency_id}/{carpool_id}.json") + os.remove(f"data/enhanced/{agency_id}/{carpool_id}.json", ) + except FileNotFoundError: + pass try: from amarillo.plugins.metrics import trips_deleted_counter diff --git a/amarillo/services/config.py b/amarillo/services/config.py index e55eec7..19c276b 100644 --- a/amarillo/services/config.py +++ b/amarillo/services/config.py @@ -10,6 +10,7 @@ class Config(BaseSettings): env: str = 'DEV' graphhopper_base_url: str = 'https://api.mfdz.de/gh' stop_sources_file: str = 'conf/stop_sources.json' + enhancer_url: str = 'http://localhost:8001' model_config = ConfigDict(extra='allow') # Allow plugins to add extra values From 6b42501fa89c0c622f61369be28e21c040514a74 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 18 Jul 2024 11:43:00 +0200 Subject: [PATCH 24/50] Carpool event hooks --- amarillo/routers/carpool.py | 27 ++++++++------------------- amarillo/services/carpools.py | 5 ----- amarillo/services/hooks.py | 27 +++++++++++++++++++++++++++ 3 files changed, 35 insertions(+), 24 deletions(-) create mode 100644 amarillo/services/hooks.py diff --git a/amarillo/routers/carpool.py b/amarillo/routers/carpool.py index c741aa9..08c9382 100644 --- a/amarillo/routers/carpool.py +++ b/amarillo/routers/carpool.py @@ -12,7 +12,7 @@ from amarillo.models.Carpool import Carpool from amarillo.routers.agencyconf import verify_api_key, verify_permission_for_same_agency_or_admin from amarillo.tests.sampledata import examples - +from amarillo.services.hooks import run_on_create, run_on_delete from amarillo.services.config import config from amarillo.utils.utils import assert_folder_exists @@ -51,6 +51,8 @@ async def post_carpool(background_tasks: BackgroundTasks, carpool: Carpool = Bod requesting_agency_id: str = Depends(verify_api_key)) -> Carpool: await verify_permission_for_same_agency_or_admin(carpool.agency, requesting_agency_id) + background_tasks.add_task(run_on_create, carpool) + logger.info(f"POST trip {carpool.agency}:{carpool.id}.") await assert_agency_exists(carpool.agency) @@ -90,12 +92,14 @@ async def get_carpool(agency_id: str, carpool_id: str, api_key: str = Depends(ve "description": "Carpool or agency not found"}, }, ) -async def delete_carpool(agency_id: str, carpool_id: str, requesting_agency_id: str = Depends(verify_api_key)): +async def delete_carpool(background_tasks: BackgroundTasks, agency_id: str, carpool_id: str, requesting_agency_id: str = Depends(verify_api_key)): await verify_permission_for_same_agency_or_admin(agency_id, requesting_agency_id) logger.info(f"Delete trip {agency_id}:{carpool_id}.") await assert_agency_exists(agency_id) await assert_carpool_exists(agency_id, carpool_id) + cp = await load_carpool(agency_id, carpool_id) + background_tasks.add_task(run_on_delete, cp) return await _delete_carpool(agency_id, carpool_id) @@ -111,12 +115,6 @@ async def _delete_carpool(agency_id: str, carpool_id: str): os.remove(f"data/enhanced/{agency_id}/{carpool_id}.json", ) except FileNotFoundError: pass - - try: - from amarillo.plugins.metrics import trips_deleted_counter - trips_deleted_counter.inc() - except ImportError: - pass async def store_carpool(carpool: Carpool) -> Carpool: @@ -125,17 +123,6 @@ async def store_carpool(carpool: Carpool) -> Carpool: await set_lastUpdated_if_unset(carpool) await save_carpool(carpool) - try: - from amarillo.plugins.metrics import trips_created_counter, trips_updated_counter - if(carpool_exists): - # logger.info("Incrementing trips updated") - trips_updated_counter.inc() - else: - # logger.info("Incrementing trips created") - trips_created_counter.inc() - except ImportError: - pass - return carpool async def set_lastUpdated_if_unset(carpool): @@ -176,4 +163,6 @@ async def delete_agency_carpools_older_than(agency_id, timestamp): if os.path.getmtime(carpool_file_name) < timestamp: m = re.search(r'([a-zA-Z0-9_-]+)\.json$', carpool_file_name) # TODO log deletion + cp = await load_carpool(agency_id, m[1]) + run_on_delete(cp) await _delete_carpool(agency_id, m[1]) diff --git a/amarillo/services/carpools.py b/amarillo/services/carpools.py index 7b6fd39..7790cbc 100644 --- a/amarillo/services/carpools.py +++ b/amarillo/services/carpools.py @@ -37,11 +37,6 @@ def purge_outdated_offers(self): if cp and self.is_outdated(cp): logger.info("Purge outdated offer %s", key) self.delete(cp.agency, cp.id) - try: - from amarillo.plugins.metrics import trips_deleted_counter - trips_deleted_counter.inc() - except ImportError: - pass def get(self, agency_id: str, carpool_id: str): return self.carpools.get(f"{agency_id}:{carpool_id}") diff --git a/amarillo/services/hooks.py b/amarillo/services/hooks.py new file mode 100644 index 0000000..5713585 --- /dev/null +++ b/amarillo/services/hooks.py @@ -0,0 +1,27 @@ +from typing import List +from amarillo.models.Carpool import Carpool + +class CarpoolEvents: + def on_create(cp : Carpool): + pass + def on_update(cp : Carpool): + pass + def on_delete(cp : Carpool): + pass + +carpool_event_listeners : List[CarpoolEvents] = [] + +def register_carpool_event_listener(cpe : CarpoolEvents): + carpool_event_listeners.append(cpe) + +def run_on_create(cp: Carpool): + for cpe in carpool_event_listeners: + cpe.on_create(cp) + +def run_on_update(cp: Carpool): + for cpe in carpool_event_listeners: + cpe.on_update(cp) + +def run_on_delete(cp: Carpool): + for cpe in carpool_event_listeners: + cpe.on_delete(cp) \ No newline at end of file From f85291fea95b4f36fdc58d97c56219bbf6c0f349 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 24 Jun 2024 13:33:47 +0200 Subject: [PATCH 25/50] Build amarillo-base and derived image --- Dockerfile | 4 +-- Jenkinsfile | 63 +++++++++++++++++++++++++++++++-------------- pyproject.toml | 2 +- standard.Dockerfile | 16 ++++++++++++ 4 files changed, 62 insertions(+), 23 deletions(-) create mode 100644 standard.Dockerfile diff --git a/Dockerfile b/Dockerfile index aa563ca..c2c0f76 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,10 +20,8 @@ ENV RIDE2GO_TOKEN='' EXPOSE 80 -ARG PLUGINS - COPY requirements.txt /app/requirements.txt -RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt ${PLUGINS} +RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt # set MODULE_NAME explicitly ENV MODULE_NAME=amarillo.main diff --git a/Jenkinsfile b/Jenkinsfile index 2715959..7223a30 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,17 +1,17 @@ pipeline { - agent any + agent { label 'builtin' } environment { GITEA_CREDS = credentials('AMARILLO-JENKINS-GITEA-USER') - PYPI_CREDS = credentials('AMARILLO-JENKINS-PYPI-USER') TWINE_REPO_URL = "https://git.gerhardt.io/api/packages/amarillo/pypi" PLUGINS_REPO_URL = "git.gerhardt.io/api/packages/amarillo/pypi/simple" - DOCKER_REGISTRY_URL = 'https://git.gerhardt.io' + DOCKER_REGISTRY = 'git.gerhardt.io' + DERIVED_DOCKERFILE = 'standard.Dockerfile' OWNER = 'amarillo' + BASE_IMAGE_NAME = 'amarillo-base' IMAGE_NAME = 'amarillo' - AMARILLO_DISTRIBUTION = '0.2' - TAG = "${AMARILLO_DISTRIBUTION}.${BUILD_NUMBER}" - PLUGINS = 'amarillo-metrics amarillo-enhancer amarillo-grfs-export' - DEPLOY_WEBHOOK_URL = 'http://amarillo.mfdz.de:8888/mitanand' + AMARILLO_DISTRIBUTION = '0.3' + TAG = "${AMARILLO_DISTRIBUTION}.${BUILD_NUMBER}${env.BRANCH_NAME == 'main' ? '' : '-' + env.BRANCH_NAME}" + DEPLOY_WEBHOOK_URL = "http://amarillo.mfdz.de:8888/${env.BRANCH_NAME}" DEPLOY_SECRET = credentials('AMARILLO-JENKINS-DEPLOY-SECRET') } stages { @@ -42,35 +42,56 @@ pipeline { sh 'python3 -m twine upload --skip-existing --verbose --repository-url $TWINE_REPO_URL --username $GITEA_CREDS_USR --password $GITEA_CREDS_PSW ./dist/*' } } - stage('Publish package to PyPI') { + stage('Build base docker image') { when { - branch 'release' + isDeployBranch() } steps { - sh 'python3 -m twine upload --verbose --username $PYPI_CREDS_USR --password $PYPI_CREDS_PSW ./dist/*' + echo 'Building image' + script { + docker.build("${OWNER}/${BASE_IMAGE_NAME}:${TAG}") + } + } + } + stage('Push base image to container registry') { + when { + isDeployBranch() + } + steps { + echo 'Pushing image to registry' + script { + docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){ + def image = docker.image("${OWNER}/${BASE_IMAGE_NAME}:${TAG}") + image.push() + image.push('latest') + } + } } } - stage('Build Mitanand docker image') { + stage('Build derived docker image') { when { - branch 'mitanand' + isDeployBranch() } steps { echo 'Building image' script { - docker.build("${OWNER}/${IMAGE_NAME}:${TAG}", - //--no-cache to make sure plugins are updated - "--no-cache --build-arg='PACKAGE_REGISTRY_URL=${PLUGINS_REPO_URL}' --build-arg='PLUGINS=${PLUGINS}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .") + docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){ + docker.build("${OWNER}/${IMAGE_NAME}:${TAG}", + //--no-cache to make sure plugins are updated + "-f ${DERIVED_DOCKERFILE} --no-cache --build-arg='PACKAGE_REGISTRY_URL=${PLUGINS_REPO_URL}' --build-arg='DOCKER_REGISTRY=${DOCKER_REGISTRY}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .") + } + } } } - stage('Push image to container registry') { + stage('Push derived image to container registry') { when { - branch 'mitanand' + isDeployBranch() } steps { echo 'Pushing image to registry' script { - docker.withRegistry(DOCKER_REGISTRY_URL, 'AMARILLO-JENKINS-GITEA-USER'){ + docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){ def image = docker.image("${OWNER}/${IMAGE_NAME}:${TAG}") image.push() image.push('latest') @@ -80,7 +101,7 @@ pipeline { } stage('Notify CD script') { when { - branch 'mitanand' + isDeployBranch() } steps { echo 'Triggering deploy webhook' @@ -93,3 +114,7 @@ pipeline { } } } + +def isDeployBranch() { + return anyOf { branch 'main'; branch 'dev'; branch 'mitanand' } +} diff --git a/pyproject.toml b/pyproject.toml index af5be75..6eb254e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "amarillo" -version = "0.0.15a3" +version = "0.0.17a1" description = "Aggregates and enhances carpooling-offers and publishes them as GTFS(-RT)" readme = "README.md" license = {file = "LICENSE"} diff --git a/standard.Dockerfile b/standard.Dockerfile new file mode 100644 index 0000000..3066d0b --- /dev/null +++ b/standard.Dockerfile @@ -0,0 +1,16 @@ +ARG DOCKER_REGISTRY +FROM ${DOCKER_REGISTRY}/amarillo/amarillo-base + +ARG PLUGINS=\ +"amarillo-metrics "\ +"amarillo-gtfs-exporter " + +ARG PACKAGE_REGISTRY_URL + +ENV METRICS_USER='' +ENV METRICS_PASSWORD='' + +# RUN pip install $PLUGINS + +RUN --mount=type=secret,id=AMARILLO_REGISTRY_CREDENTIALS \ +pip install --no-cache-dir --upgrade --extra-index-url https://$(cat /run/secrets/AMARILLO_REGISTRY_CREDENTIALS)@${PACKAGE_REGISTRY_URL} ${PLUGINS} From b1ec8e1a9293c5224dabc3df8d5501783e290a96 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 5 Aug 2024 15:56:24 +0200 Subject: [PATCH 26/50] Publish to PyPI, removed conditionals in Jenkinsfile --- Jenkinsfile | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7223a30..fed39c1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,6 +2,7 @@ pipeline { agent { label 'builtin' } environment { GITEA_CREDS = credentials('AMARILLO-JENKINS-GITEA-USER') + PYPI_CREDS = credentials('AMARILLO-JENKINS-PYPI-USER') TWINE_REPO_URL = "https://git.gerhardt.io/api/packages/amarillo/pypi" PLUGINS_REPO_URL = "git.gerhardt.io/api/packages/amarillo/pypi/simple" DOCKER_REGISTRY = 'git.gerhardt.io' @@ -11,7 +12,7 @@ pipeline { IMAGE_NAME = 'amarillo' AMARILLO_DISTRIBUTION = '0.3' TAG = "${AMARILLO_DISTRIBUTION}.${BUILD_NUMBER}${env.BRANCH_NAME == 'main' ? '' : '-' + env.BRANCH_NAME}" - DEPLOY_WEBHOOK_URL = "http://amarillo.mfdz.de:8888/${env.BRANCH_NAME}" + DEPLOY_WEBHOOK_URL = "http://amarillo.mfdz.de:8888/dev" DEPLOY_SECRET = credentials('AMARILLO-JENKINS-DEPLOY-SECRET') } stages { @@ -42,10 +43,15 @@ pipeline { sh 'python3 -m twine upload --skip-existing --verbose --repository-url $TWINE_REPO_URL --username $GITEA_CREDS_USR --password $GITEA_CREDS_PSW ./dist/*' } } - stage('Build base docker image') { + stage('Publish package to PyPI') { when { - isDeployBranch() + branch 'main' } + steps { + sh 'python3 -m twine upload --verbose --username $PYPI_CREDS_USR --password $PYPI_CREDS_PSW ./dist/*' + } + } + stage('Build base docker image') { steps { echo 'Building image' script { @@ -54,9 +60,6 @@ pipeline { } } stage('Push base image to container registry') { - when { - isDeployBranch() - } steps { echo 'Pushing image to registry' script { @@ -69,9 +72,6 @@ pipeline { } } stage('Build derived docker image') { - when { - isDeployBranch() - } steps { echo 'Building image' script { @@ -85,9 +85,6 @@ pipeline { } } stage('Push derived image to container registry') { - when { - isDeployBranch() - } steps { echo 'Pushing image to registry' script { @@ -101,7 +98,9 @@ pipeline { } stage('Notify CD script') { when { - isDeployBranch() + not { + branch 'main' + } } steps { echo 'Triggering deploy webhook' @@ -113,8 +112,4 @@ pipeline { } } } -} - -def isDeployBranch() { - return anyOf { branch 'main'; branch 'dev'; branch 'mitanand' } -} +} \ No newline at end of file From fac12e4b1cd2a672667a653f8056db6eba5fbcac Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 6 Aug 2024 10:37:25 +0200 Subject: [PATCH 27/50] Removed credential requirement from standard.Dockerfile --- Jenkinsfile | 5 +++-- standard.Dockerfile | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fed39c1..736d980 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,7 +4,8 @@ pipeline { GITEA_CREDS = credentials('AMARILLO-JENKINS-GITEA-USER') PYPI_CREDS = credentials('AMARILLO-JENKINS-PYPI-USER') TWINE_REPO_URL = "https://git.gerhardt.io/api/packages/amarillo/pypi" - PLUGINS_REPO_URL = "git.gerhardt.io/api/packages/amarillo/pypi/simple" + PLUGINS_REPO_URL = "https://git.gerhardt.io/api/packages/amarillo/pypi/simple" + PYPI_REPO_URL = "https://pypi.org/simple" DOCKER_REGISTRY = 'git.gerhardt.io' DERIVED_DOCKERFILE = 'standard.Dockerfile' OWNER = 'amarillo' @@ -78,7 +79,7 @@ pipeline { docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){ docker.build("${OWNER}/${IMAGE_NAME}:${TAG}", //--no-cache to make sure plugins are updated - "-f ${DERIVED_DOCKERFILE} --no-cache --build-arg='PACKAGE_REGISTRY_URL=${PLUGINS_REPO_URL}' --build-arg='DOCKER_REGISTRY=${DOCKER_REGISTRY}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .") + "-f ${DERIVED_DOCKERFILE} --no-cache --build-arg='PACKAGE_REGISTRY_URL=${env.BRANCH_NAME == 'main' ? env.PYPI_REPO_URL : env.PLUGINS_REPO_URL}' --build-arg='DOCKER_REGISTRY=${DOCKER_REGISTRY}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .") } } diff --git a/standard.Dockerfile b/standard.Dockerfile index 3066d0b..52cf951 100644 --- a/standard.Dockerfile +++ b/standard.Dockerfile @@ -12,5 +12,4 @@ ENV METRICS_PASSWORD='' # RUN pip install $PLUGINS -RUN --mount=type=secret,id=AMARILLO_REGISTRY_CREDENTIALS \ -pip install --no-cache-dir --upgrade --extra-index-url https://$(cat /run/secrets/AMARILLO_REGISTRY_CREDENTIALS)@${PACKAGE_REGISTRY_URL} ${PLUGINS} +RUN pip install --no-cache-dir --upgrade --extra-index-url ${PACKAGE_REGISTRY_URL} ${PLUGINS} From ea4af99f833b52c846d5cff794bb706f5df3cf9c Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 6 Aug 2024 11:58:34 +0200 Subject: [PATCH 28/50] Changed Jenkinsfile distribution version to 2.0.0 --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 736d980..c29789a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -11,8 +11,8 @@ pipeline { OWNER = 'amarillo' BASE_IMAGE_NAME = 'amarillo-base' IMAGE_NAME = 'amarillo' - AMARILLO_DISTRIBUTION = '0.3' - TAG = "${AMARILLO_DISTRIBUTION}.${BUILD_NUMBER}${env.BRANCH_NAME == 'main' ? '' : '-' + env.BRANCH_NAME}" + AMARILLO_DISTRIBUTION = '2.0.0' + TAG = "${AMARILLO_DISTRIBUTION}${env.BRANCH_NAME == 'main' ? '' : '-' + env.BRANCH_NAME}-${BUILD_NUMBER}" DEPLOY_WEBHOOK_URL = "http://amarillo.mfdz.de:8888/dev" DEPLOY_SECRET = credentials('AMARILLO-JENKINS-DEPLOY-SECRET') } From 54e25b31bcb304fc2d8531971dd97ba3c24cf105 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 6 Aug 2024 13:32:32 +0200 Subject: [PATCH 29/50] Run as amarillo user --- standard.Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/standard.Dockerfile b/standard.Dockerfile index 52cf951..d4a91b6 100644 --- a/standard.Dockerfile +++ b/standard.Dockerfile @@ -13,3 +13,6 @@ ENV METRICS_PASSWORD='' # RUN pip install $PLUGINS RUN pip install --no-cache-dir --upgrade --extra-index-url ${PACKAGE_REGISTRY_URL} ${PLUGINS} + +RUN useradd amarillo +USER amarillo \ No newline at end of file From 04f8fae9d4ba5581ff7a73b7f17df221883e767a Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 6 Aug 2024 13:47:59 +0200 Subject: [PATCH 30/50] Disable 'notify CD script' stage --- Jenkinsfile | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index c29789a..c024bbd 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -97,20 +97,20 @@ pipeline { } } } - stage('Notify CD script') { - when { - not { - branch 'main' - } - } - steps { - echo 'Triggering deploy webhook' - script { - def response = httpRequest contentType: 'APPLICATION_JSON', - httpMode: 'POST', requestBody: '{}', authentication: 'AMARILLO-JENKINS-DEPLOY-SECRET', - url: "${DEPLOY_WEBHOOK_URL}" - } - } - } + // stage('Notify CD script') { + // when { + // not { + // branch 'main' + // } + // } + // steps { + // echo 'Triggering deploy webhook' + // script { + // def response = httpRequest contentType: 'APPLICATION_JSON', + // httpMode: 'POST', requestBody: '{}', authentication: 'AMARILLO-JENKINS-DEPLOY-SECRET', + // url: "${DEPLOY_WEBHOOK_URL}" + // } + // } + // } } } \ No newline at end of file From 5c9904b0500e3f09f588cff9891b76a89015391a Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 6 Aug 2024 13:50:31 +0200 Subject: [PATCH 31/50] Fixed error.log permissions --- standard.Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/standard.Dockerfile b/standard.Dockerfile index d4a91b6..d942f74 100644 --- a/standard.Dockerfile +++ b/standard.Dockerfile @@ -14,5 +14,9 @@ ENV METRICS_PASSWORD='' RUN pip install --no-cache-dir --upgrade --extra-index-url ${PACKAGE_REGISTRY_URL} ${PLUGINS} +# Create the error.log, otherwise we get a permission error when we try to write to it +RUN touch /app/error.log +RUN chmod 777 /app/error.log + RUN useradd amarillo USER amarillo \ No newline at end of file From f58a7392b26481f253ab037ef0c3f02cf7ae4b41 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 6 Aug 2024 13:55:55 +0200 Subject: [PATCH 32/50] Set version to 2.0.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6eb254e..b284faa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "amarillo" -version = "0.0.17a1" +version = "2.0.0" description = "Aggregates and enhances carpooling-offers and publishes them as GTFS(-RT)" readme = "README.md" license = {file = "LICENSE"} From 912bd56a7c207c6ccc95cc820029f09eac9f5301 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Fri, 17 May 2024 13:20:16 +0200 Subject: [PATCH 33/50] Use /data for region and agency configuration --- .gitignore | 1 + Dockerfile | 2 +- amarillo/main.py | 2 +- amarillo/routers/carpool.py | 2 +- amarillo/services/agencies.py | 3 +-- amarillo/services/config.py | 2 +- amarillo/services/regions.py | 3 +-- amarillo/static/{conf => data}/agency/mfdz.json | 0 amarillo/static/{conf => data}/agency/mifaz.json | 0 amarillo/static/{conf => data}/agency/ride2go.json | 0 amarillo/static/{conf => data}/agency/ummadum.json | 0 amarillo/static/{conf => data}/region/bb.json | 0 amarillo/static/{conf => data}/region/bw.json | 0 amarillo/static/{conf => data}/region/by.json | 0 amarillo/static/{conf => data}/region/nrw.json | 0 amarillo/static/{conf => data}/stop_sources.json | 0 16 files changed, 7 insertions(+), 8 deletions(-) rename amarillo/static/{conf => data}/agency/mfdz.json (100%) rename amarillo/static/{conf => data}/agency/mifaz.json (100%) rename amarillo/static/{conf => data}/agency/ride2go.json (100%) rename amarillo/static/{conf => data}/agency/ummadum.json (100%) rename amarillo/static/{conf => data}/region/bb.json (100%) rename amarillo/static/{conf => data}/region/bw.json (100%) rename amarillo/static/{conf => data}/region/by.json (100%) rename amarillo/static/{conf => data}/region/nrw.json (100%) rename amarillo/static/{conf => data}/stop_sources.json (100%) diff --git a/.gitignore b/.gitignore index 6d0e3dc..e3f2885 100644 --- a/.gitignore +++ b/.gitignore @@ -143,6 +143,7 @@ data/failed/ data/trash/ data/gtfs/ data/tmp +data/** #these files are under app/static but they get copied to the outside directory on startup logging.conf diff --git a/Dockerfile b/Dockerfile index c2c0f76..6a0bc14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,7 @@ COPY ./amarillo/static/static /app/static COPY ./amarillo/static/templates /app/templates COPY ./amarillo/static/config /app COPY ./amarillo/static/logging.conf /app -COPY ./amarillo/static/conf /app/conf +COPY ./amarillo/static/data /app/data # This image inherits uvicorn-gunicorn's CMD. If you'd like to start uvicorn, use this instead # CMD ["uvicorn", "amarillo.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/amarillo/main.py b/amarillo/main.py index 19b4ec5..7f1d73b 100644 --- a/amarillo/main.py +++ b/amarillo/main.py @@ -7,7 +7,7 @@ from amarillo.utils.utils import copy_static_files #this has to run before app.configuration is imported, otherwise we get validation error for config because the config file is not copied yet -copy_static_files(["conf", "static", "templates", "logging.conf", "config"]) +copy_static_files(["data", "static", "templates", "logging.conf", "config"]) import amarillo.plugins from amarillo.services.config import config diff --git a/amarillo/routers/carpool.py b/amarillo/routers/carpool.py index 08c9382..4be7d8e 100644 --- a/amarillo/routers/carpool.py +++ b/amarillo/routers/carpool.py @@ -143,7 +143,7 @@ async def save_carpool(carpool, folder: str = 'data/carpool'): async def assert_agency_exists(agency_id: str): - agency_exists = os.path.exists(f"conf/agency/{agency_id}.json") + agency_exists = os.path.exists(f"data/agency/{agency_id}.json") if not agency_exists: raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, diff --git a/amarillo/services/agencies.py b/amarillo/services/agencies.py index 47baa5a..e7450aa 100644 --- a/amarillo/services/agencies.py +++ b/amarillo/services/agencies.py @@ -12,8 +12,7 @@ class AgencyService: def __init__(self): self.agencies: Dict[str, Agency] = {} - - for agency_file_name in glob('conf/agency/*.json'): + for agency_file_name in glob('data/agency/*.json'): with open(agency_file_name) as agency_file: dict = json.load(agency_file) agency = Agency(**dict) diff --git a/amarillo/services/config.py b/amarillo/services/config.py index 19c276b..867c169 100644 --- a/amarillo/services/config.py +++ b/amarillo/services/config.py @@ -9,7 +9,7 @@ class Config(BaseSettings): ride2go_query_data: str env: str = 'DEV' graphhopper_base_url: str = 'https://api.mfdz.de/gh' - stop_sources_file: str = 'conf/stop_sources.json' + stop_sources_file: str = 'data/stop_sources.json' enhancer_url: str = 'http://localhost:8001' model_config = ConfigDict(extra='allow') # Allow plugins to add extra values diff --git a/amarillo/services/regions.py b/amarillo/services/regions.py index 8be79e0..425b3ac 100644 --- a/amarillo/services/regions.py +++ b/amarillo/services/regions.py @@ -9,8 +9,7 @@ class RegionService: def __init__(self): self.regions: Dict[str, Region] = {} - - for region_file_name in glob('conf/region/*.json'): + for region_file_name in glob('data/region/*.json'): with open(region_file_name) as region_file: dict = json.load(region_file) region = Region(**dict) diff --git a/amarillo/static/conf/agency/mfdz.json b/amarillo/static/data/agency/mfdz.json similarity index 100% rename from amarillo/static/conf/agency/mfdz.json rename to amarillo/static/data/agency/mfdz.json diff --git a/amarillo/static/conf/agency/mifaz.json b/amarillo/static/data/agency/mifaz.json similarity index 100% rename from amarillo/static/conf/agency/mifaz.json rename to amarillo/static/data/agency/mifaz.json diff --git a/amarillo/static/conf/agency/ride2go.json b/amarillo/static/data/agency/ride2go.json similarity index 100% rename from amarillo/static/conf/agency/ride2go.json rename to amarillo/static/data/agency/ride2go.json diff --git a/amarillo/static/conf/agency/ummadum.json b/amarillo/static/data/agency/ummadum.json similarity index 100% rename from amarillo/static/conf/agency/ummadum.json rename to amarillo/static/data/agency/ummadum.json diff --git a/amarillo/static/conf/region/bb.json b/amarillo/static/data/region/bb.json similarity index 100% rename from amarillo/static/conf/region/bb.json rename to amarillo/static/data/region/bb.json diff --git a/amarillo/static/conf/region/bw.json b/amarillo/static/data/region/bw.json similarity index 100% rename from amarillo/static/conf/region/bw.json rename to amarillo/static/data/region/bw.json diff --git a/amarillo/static/conf/region/by.json b/amarillo/static/data/region/by.json similarity index 100% rename from amarillo/static/conf/region/by.json rename to amarillo/static/data/region/by.json diff --git a/amarillo/static/conf/region/nrw.json b/amarillo/static/data/region/nrw.json similarity index 100% rename from amarillo/static/conf/region/nrw.json rename to amarillo/static/data/region/nrw.json diff --git a/amarillo/static/conf/stop_sources.json b/amarillo/static/data/stop_sources.json similarity index 100% rename from amarillo/static/conf/stop_sources.json rename to amarillo/static/data/stop_sources.json From e31c09708e5ec910ed414da6f817a293681e51cd Mon Sep 17 00:00:00 2001 From: Holger Bruch Date: Mon, 6 May 2024 18:14:12 +0200 Subject: [PATCH 34/50] fix: call _extract_stop on class --- amarillo/services/importing/ride2go.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/amarillo/services/importing/ride2go.py b/amarillo/services/importing/ride2go.py index 098411c..187ef26 100644 --- a/amarillo/services/importing/ride2go.py +++ b/amarillo/services/importing/ride2go.py @@ -29,7 +29,7 @@ def _extract_carpool(dict) -> Carpool: id=id, agency=agency, deeplink=dict['deeplink'], - stops=[self._extract_stop(s) for s in dict.get('stops')], + stops=[Ride2GoImporter._extract_stop(s) for s in dict.get('stops')], departureTime=dict.get('departTime'), departureDate=dict.get('departDate') if dict.get('departDate') else dict.get('weekdays'), lastUpdated=dict.get('lastUpdated'), From e2253a8902da7474f896fa45bbaddddc288e50d4 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Wed, 14 Aug 2024 11:37:12 +0200 Subject: [PATCH 35/50] Enhance synced trips --- amarillo/routers/agency.py | 10 +++++++--- amarillo/routers/carpool.py | 19 ++++++++++++------- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/amarillo/routers/agency.py b/amarillo/routers/agency.py index 5b09db4..67e98c6 100644 --- a/amarillo/routers/agency.py +++ b/amarillo/routers/agency.py @@ -2,12 +2,12 @@ import time from typing import List -from fastapi import APIRouter, HTTPException, status, Depends +from fastapi import APIRouter, HTTPException, status, Depends, BackgroundTasks from amarillo.models.Carpool import Carpool, Agency from amarillo.routers.agencyconf import verify_api_key, verify_admin_api_key, verify_permission_for_same_agency_or_admin # TODO should move this to service -from amarillo.routers.carpool import store_carpool, delete_agency_carpools_older_than +from amarillo.routers.carpool import store_carpool, delete_agency_carpools_older_than, enhance_trip from amarillo.services.agencies import AgencyService from amarillo.services.importing import Ride2GoImporter, NoiImporter from amarillo.utils.container import container @@ -61,7 +61,7 @@ async def get_agency(agency_id: str, admin_api_key: str = Depends(verify_api_key status.HTTP_500_INTERNAL_SERVER_ERROR: { "description": "Import error"} }) -async def sync(agency_id: str, requesting_agency_id: str = Depends(verify_api_key)) -> List[Carpool]: +async def sync(background_tasks: BackgroundTasks, agency_id: str, requesting_agency_id: str = Depends(verify_api_key)) -> List[Carpool]: await verify_permission_for_same_agency_or_admin(agency_id, requesting_agency_id) if agency_id == "ride2go": @@ -78,6 +78,10 @@ async def sync(agency_id: str, requesting_agency_id: str = Depends(verify_api_ke # Reduce current time by a minute to avoid inter process timestamp issues synced_files_older_than = time.time() - 60 result = [await store_carpool(cp) for cp in carpools] + + for cp in carpools: + background_tasks.add_task(enhance_trip, cp) + await delete_agency_carpools_older_than(agency_id, synced_files_older_than) return result except BaseException as e: diff --git a/amarillo/routers/carpool.py b/amarillo/routers/carpool.py index 4be7d8e..b4c31fa 100644 --- a/amarillo/routers/carpool.py +++ b/amarillo/routers/carpool.py @@ -26,15 +26,20 @@ #TODO: housekeeping for outdated trips def enhance_trip(carpool: Carpool): - response = requests.post(f"{config.enhancer_url}", carpool.model_dump_json()) - enhanced_carpool = Carpool(**json.loads(response.content)) + try: + response = requests.post(f"{config.enhancer_url}", carpool.model_dump_json()) + response.raise_for_status() + enhanced_carpool = Carpool(**json.loads(response.content)) + + folder = f'data/enhanced/{carpool.agency}' + filename = f'{folder}/{carpool.id}.json' - folder = f'data/enhanced/{carpool.agency}' - filename = f'{folder}/{carpool.id}.json' + assert_folder_exists(folder) + with open(filename, 'w', encoding='utf-8') as f: + f.write(enhanced_carpool.model_dump_json()) + except Exception as e: + logger.error(f"Error during call to enhancer: {e}") - assert_folder_exists(folder) - with open(filename, 'w', encoding='utf-8') as f: - f.write(enhanced_carpool.model_dump_json()) @router.post("/", operation_id="addcarpool", From 6235798562a9faa3f166ec03d836c985b696d724 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Wed, 14 Aug 2024 12:21:04 +0200 Subject: [PATCH 36/50] fix: encode request body as utf-8 --- amarillo/routers/carpool.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/amarillo/routers/carpool.py b/amarillo/routers/carpool.py index b4c31fa..65e399e 100644 --- a/amarillo/routers/carpool.py +++ b/amarillo/routers/carpool.py @@ -27,7 +27,7 @@ def enhance_trip(carpool: Carpool): try: - response = requests.post(f"{config.enhancer_url}", carpool.model_dump_json()) + response = requests.post(f"{config.enhancer_url}", carpool.model_dump_json().encode('utf-8')) response.raise_for_status() enhanced_carpool = Carpool(**json.loads(response.content)) @@ -38,7 +38,7 @@ def enhance_trip(carpool: Carpool): with open(filename, 'w', encoding='utf-8') as f: f.write(enhanced_carpool.model_dump_json()) except Exception as e: - logger.error(f"Error during call to enhancer: {e}") + logger.error(f"Error enhancing trip '{carpool.id}': {e}") @router.post("/", From b9321444162a16f5c3d54c862199f166b4e284c9 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Wed, 30 Oct 2024 13:04:02 +0100 Subject: [PATCH 37/50] Add Mitanand to server list --- amarillo/main.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/amarillo/main.py b/amarillo/main.py index 7f1d73b..1d3c34e 100644 --- a/amarillo/main.py +++ b/amarillo/main.py @@ -70,6 +70,10 @@ "description": "Demo server by MFDZ", "url": "https://amarillo.mfdz.de" }, + { + "description": "Mitanand Amarillo service", + "url": "https://mitanand.mfdz.de" + }, { "description": "Dev server for development", "url": "https://amarillo-dev.mfdz.de" From af2f7c332b27c3faeb5b72b9790528aa784d8c63 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 21 Nov 2024 13:20:04 +0100 Subject: [PATCH 38/50] Log user info why API key is used --- amarillo/routers/agencyconf.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/amarillo/routers/agencyconf.py b/amarillo/routers/agencyconf.py index 036cd95..006162d 100644 --- a/amarillo/routers/agencyconf.py +++ b/amarillo/routers/agencyconf.py @@ -36,7 +36,9 @@ async def verify_admin_api_key(X_API_Key: str = Header(...)): async def verify_api_key(X_API_Key: str = Header(...)): agency_conf_service: AgencyConfService = container['agencyconf'] - return agency_conf_service.check_api_key(X_API_Key) + agency_id = agency_conf_service.check_api_key(X_API_Key) + logger.info(f"API key used: {agency_id}") + return agency_id # TODO Return code 403 Unauthoized (in response_status_codes as well...) async def verify_permission_for_same_agency_or_admin(agency_id_in_path_or_body, agency_id_from_api_key): From b575b64aa8f162cc6d9715ab1c1e500bdfd4770a Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 25 Nov 2024 12:19:36 +0100 Subject: [PATCH 39/50] Build mitanand image --- Jenkinsfile | 28 ++++++++++++++++++++++++++++ mitanand.Dockerfile | 22 ++++++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 mitanand.Dockerfile diff --git a/Jenkinsfile b/Jenkinsfile index c024bbd..6ccf05f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -8,9 +8,11 @@ pipeline { PYPI_REPO_URL = "https://pypi.org/simple" DOCKER_REGISTRY = 'git.gerhardt.io' DERIVED_DOCKERFILE = 'standard.Dockerfile' + MITANAND_DOCKERFILE = 'mitanand.Dockerfile' OWNER = 'amarillo' BASE_IMAGE_NAME = 'amarillo-base' IMAGE_NAME = 'amarillo' + MITANAND_IMAGE_NAME = 'amarillo-mitanand' AMARILLO_DISTRIBUTION = '2.0.0' TAG = "${AMARILLO_DISTRIBUTION}${env.BRANCH_NAME == 'main' ? '' : '-' + env.BRANCH_NAME}-${BUILD_NUMBER}" DEPLOY_WEBHOOK_URL = "http://amarillo.mfdz.de:8888/dev" @@ -97,6 +99,32 @@ pipeline { } } } + + stage('Build mitanand docker image') { + steps { + echo 'Building image' + script { + docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){ + docker.build("${OWNER}/${MITANAND_IMAGE_NAME}:${TAG}", + //--no-cache to make sure plugins are updated + "-f ${MITANAND_DOCKERFILE} --no-cache --build-arg='PACKAGE_REGISTRY_URL=${env.BRANCH_NAME == 'main' ? env.PYPI_REPO_URL : env.PLUGINS_REPO_URL}' --build-arg='DOCKER_REGISTRY=${DOCKER_REGISTRY}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .") + } + + } + } + } + stage('Push derived image to container registry') { + steps { + echo 'Pushing image to registry' + script { + docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){ + def image = docker.image("${OWNER}/${MITANAND_IMAGE_NAME}:${TAG}") + image.push() + image.push('latest') + } + } + } + } // stage('Notify CD script') { // when { // not { diff --git a/mitanand.Dockerfile b/mitanand.Dockerfile new file mode 100644 index 0000000..de7543a --- /dev/null +++ b/mitanand.Dockerfile @@ -0,0 +1,22 @@ +ARG DOCKER_REGISTRY +FROM ${DOCKER_REGISTRY}/amarillo/amarillo-base + +ARG PLUGINS=\ +"amarillo-metrics "\ +"amarillo-grfs-exporter " + +ARG PACKAGE_REGISTRY_URL + +ENV METRICS_USER='' +ENV METRICS_PASSWORD='' + +# RUN pip install $PLUGINS + +RUN pip install --no-cache-dir --upgrade --extra-index-url ${PACKAGE_REGISTRY_URL} ${PLUGINS} + +# Create the error.log, otherwise we get a permission error when we try to write to it +RUN touch /app/error.log +RUN chmod 777 /app/error.log + +RUN useradd amarillo +USER amarillo \ No newline at end of file From b739eb76725e6f5a5c1a8c3c114b0d5f5a6b98b0 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Mon, 25 Nov 2024 12:24:14 +0100 Subject: [PATCH 40/50] Fix mitanand jenkins stage name --- Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile b/Jenkinsfile index 6ccf05f..062a50b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -113,7 +113,7 @@ pipeline { } } } - stage('Push derived image to container registry') { + stage('Push mitanand image to container registry') { steps { echo 'Pushing image to registry' script { From 818999dbbc97c8c31d55495d9fd34cf7ed679159 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Fri, 13 Dec 2024 10:46:24 +0100 Subject: [PATCH 41/50] Plugin development readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 15f1fe9..c0c5990 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,10 @@ $ protoc --proto_path=. --python_out=../services/gtfsrt gtfs-realtime.proto real $ sed 's/import gtfs_realtime_pb2/import amarillo.services.gtfsrt.gtfs_realtime_pb2/g' ../services/gtfsrt/realtime_extension_pb2.py | sponge ../services/gtfsrt/realtime_extension_pb2.py ``` +### Develop Amarillo and its plugins together + +To the develop Amarillo and its plugins concurrently, clone this repo and all of the repos you would like to make changes to. Install the local version of the plugin(s) into the virtual environment, simply using `pip install /path/to/plugin`, or `pip install -e /path/to/plugin` if your environment supports editable installs. + ## Testing In the top directory, run `pytest amarillo/tests`. From d783f8b091bd0f3618038ce232dd5a4251c4bf8c Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Fri, 13 Dec 2024 11:17:27 +0100 Subject: [PATCH 42/50] Update logging.conf to save errors to file --- amarillo/static/logging.conf | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/amarillo/static/logging.conf b/amarillo/static/logging.conf index 429da8e..142287a 100644 --- a/amarillo/static/logging.conf +++ b/amarillo/static/logging.conf @@ -2,14 +2,14 @@ keys=root [handlers] -keys=consoleHandler +keys=consoleHandler, fileHandler [formatters] keys=simpleFormatter [logger_root] level=INFO -handlers=consoleHandler +handlers=consoleHandler, fileHandler propagate=yes [handler_consoleHandler] @@ -18,5 +18,11 @@ level=DEBUG formatter=simpleFormatter args=(sys.stdout,) +[handler_fileHandler] +class=handlers.RotatingFileHandler +level=ERROR +formatter=simpleFormatter +args=('error.log', 'a', 1000000, 3) # Filename, mode, maxBytes, backupCount + [formatter_simpleFormatter] -format=%(asctime)s - %(name)s - %(levelname)s - %(message)s \ No newline at end of file +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s From 71942e852f604fbeeb52624757fff8934bb206bb Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Thu, 19 Dec 2024 13:55:38 +0100 Subject: [PATCH 43/50] Added make_sample_data script --- make_sample_data.sh | 168 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100755 make_sample_data.sh diff --git a/make_sample_data.sh b/make_sample_data.sh new file mode 100755 index 0000000..51c9edf --- /dev/null +++ b/make_sample_data.sh @@ -0,0 +1,168 @@ +#!/bin/sh + +# 1. create user with random api key printed to console + +file=data/agencyconf/sampleuser.json +if [ -e "$file" ]; then + echo "Sample user already exists" +else + # https://security.stackexchange.com/questions/183948/unix-command-to-generate-cryptographically-secure-random-string + api_key=$(LC_ALL=C tr -dc '[:alnum:]' < /dev/urandom | head -c20) + + sampleuser_json=`cat < $file && echo "sampleuser created with api key '$api_key'" +fi + +# 2. create "sampleagency" + +file=data/agency/sample.json +if [ -e "$file" ]; then + echo "Sample agency already exists" +else + agency_id="sample" + + sampleagency_json=`cat < data/agency/sample.json && echo "Created sampleagency" +fi + +# ------ CREATE TRIPS ----- + +now=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + +# Stuttgart to Berlin $today at 18:00 +trip_id="1001" + +trip_json=`cat < "data/carpool/$agency_id/$trip_id.json" && echo "Created trip $trip_id (Stuttgart to Berlin today at 18:00)" + +# -------- +# Stuttgart to Frankfurt $tomorrow at 18:00 +trip_id="1002" + +trip_json=`cat < "data/carpool/$agency_id/$trip_id.json" && echo "Created trip $trip_id (Stuttgart to Frankfurt tomorrow at 18:00)" + +# -------- +# Stuttgart to Herrenberg every weekday 7:00 +trip_id="1003" + +trip_json=`cat < "data/carpool/$agency_id/$trip_id.json" && echo "Created trip $trip_id (Stuttgart to Herrenberg every weekday 7:00)" + +# -------- +# Stuttgart to München one week ahead at 12:00 +trip_id="1004" + +trip_json=`cat < "data/carpool/$agency_id/$trip_id.json" && echo "Created trip $trip_id (Stuttgart to München one week ahead at 12:00)" From 835db9a119f5be66e5a60f208fb04a2f6b6d04a7 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Wed, 22 Jan 2025 10:22:29 +0100 Subject: [PATCH 44/50] Revert "Added make_sample_data script" (Adding to amarillo-compose instead) --- make_sample_data.sh | 168 -------------------------------------------- 1 file changed, 168 deletions(-) delete mode 100755 make_sample_data.sh diff --git a/make_sample_data.sh b/make_sample_data.sh deleted file mode 100755 index 51c9edf..0000000 --- a/make_sample_data.sh +++ /dev/null @@ -1,168 +0,0 @@ -#!/bin/sh - -# 1. create user with random api key printed to console - -file=data/agencyconf/sampleuser.json -if [ -e "$file" ]; then - echo "Sample user already exists" -else - # https://security.stackexchange.com/questions/183948/unix-command-to-generate-cryptographically-secure-random-string - api_key=$(LC_ALL=C tr -dc '[:alnum:]' < /dev/urandom | head -c20) - - sampleuser_json=`cat < $file && echo "sampleuser created with api key '$api_key'" -fi - -# 2. create "sampleagency" - -file=data/agency/sample.json -if [ -e "$file" ]; then - echo "Sample agency already exists" -else - agency_id="sample" - - sampleagency_json=`cat < data/agency/sample.json && echo "Created sampleagency" -fi - -# ------ CREATE TRIPS ----- - -now=$(date -u +"%Y-%m-%dT%H:%M:%SZ") - -# Stuttgart to Berlin $today at 18:00 -trip_id="1001" - -trip_json=`cat < "data/carpool/$agency_id/$trip_id.json" && echo "Created trip $trip_id (Stuttgart to Berlin today at 18:00)" - -# -------- -# Stuttgart to Frankfurt $tomorrow at 18:00 -trip_id="1002" - -trip_json=`cat < "data/carpool/$agency_id/$trip_id.json" && echo "Created trip $trip_id (Stuttgart to Frankfurt tomorrow at 18:00)" - -# -------- -# Stuttgart to Herrenberg every weekday 7:00 -trip_id="1003" - -trip_json=`cat < "data/carpool/$agency_id/$trip_id.json" && echo "Created trip $trip_id (Stuttgart to Herrenberg every weekday 7:00)" - -# -------- -# Stuttgart to München one week ahead at 12:00 -trip_id="1004" - -trip_json=`cat < "data/carpool/$agency_id/$trip_id.json" && echo "Created trip $trip_id (Stuttgart to München one week ahead at 12:00)" From 63edfc2734e1f3e8a79cdca514479a4e3148baa7 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Wed, 22 Jan 2025 10:53:29 +0100 Subject: [PATCH 45/50] Enhance missing carpools, ConnectionError message --- amarillo/configuration.py | 2 ++ amarillo/routers/carpool.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/amarillo/configuration.py b/amarillo/configuration.py index 019508f..7c42196 100644 --- a/amarillo/configuration.py +++ b/amarillo/configuration.py @@ -1,4 +1,5 @@ # separate file so that it can be imported without initializing FastAPI +from amarillo.routers.carpool import enhance_missing_carpools from amarillo.utils.container import container import logging @@ -41,6 +42,7 @@ def configure_services(): logger.info("Loaded %d regions", len(container['regions'].regions)) create_required_directories() + enhance_missing_carpools() def configure_admin_token(): if config.admin_token is None: diff --git a/amarillo/routers/carpool.py b/amarillo/routers/carpool.py index 65e399e..582f4ef 100644 --- a/amarillo/routers/carpool.py +++ b/amarillo/routers/carpool.py @@ -7,8 +7,10 @@ from fastapi import APIRouter, Body, Header, HTTPException, status, Depends, BackgroundTasks import requests +from requests.exceptions import ConnectionError from datetime import datetime +from amarillo.utils.container import container from amarillo.models.Carpool import Carpool from amarillo.routers.agencyconf import verify_api_key, verify_permission_for_same_agency_or_admin from amarillo.tests.sampledata import examples @@ -37,10 +39,29 @@ def enhance_trip(carpool: Carpool): assert_folder_exists(folder) with open(filename, 'w', encoding='utf-8') as f: f.write(enhanced_carpool.model_dump_json()) + except ConnectionError: + logger.error("Could not connect to enhancer: make sure amarillo-enhancer is running and your ENHANCER_URL environment variable is configured correctly") except Exception as e: logger.error(f"Error enhancing trip '{carpool.id}': {e}") +def enhance_missing_carpools(): + logger.info(f"Enhancing missing restored trips...") + for agency_id in container['agencies'].agencies: + for carpool_file_name in glob(f'data/carpool/{agency_id}/*.json'): + + #If the same carpool is not found in /enhanced + if not os.path.isfile(carpool_file_name.replace("/carpool", "/enhanced")): + # TODO: also enhance carpools where the enhanced version is outdated? + logger.info(f"Enhancing restored trip {carpool_file_name}") + try: + with open(carpool_file_name) as carpool_file: + carpool = Carpool(**(json.load(carpool_file))) + enhance_trip(carpool) + except Exception as e: + logger.warning("Issue during restore of carpool %s: %s", carpool_file_name, repr(e)) + + @router.post("/", operation_id="addcarpool", summary="Add a new or update existing carpool", From 475b224462f0844d7d4767ce5c490c3f3e45ed72 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Tue, 28 Jan 2025 15:20:11 +0100 Subject: [PATCH 46/50] Enhance updated carpools on startup --- amarillo/routers/carpool.py | 40 +++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/amarillo/routers/carpool.py b/amarillo/routers/carpool.py index 582f4ef..a896ea0 100644 --- a/amarillo/routers/carpool.py +++ b/amarillo/routers/carpool.py @@ -2,6 +2,7 @@ import json import os import os.path +from pathlib import Path import re from glob import glob @@ -44,23 +45,38 @@ def enhance_trip(carpool: Carpool): except Exception as e: logger.error(f"Error enhancing trip '{carpool.id}': {e}") - def enhance_missing_carpools(): logger.info(f"Enhancing missing restored trips...") for agency_id in container['agencies'].agencies: for carpool_file_name in glob(f'data/carpool/{agency_id}/*.json'): - - #If the same carpool is not found in /enhanced - if not os.path.isfile(carpool_file_name.replace("/carpool", "/enhanced")): - # TODO: also enhance carpools where the enhanced version is outdated? + carpool_id = Path(carpool_file_name).stem + carpool = _load_carpool_if_exists(agency_id, carpool_id, folder='data/carpool') + if not carpool: + # logger.warning(f"Failed loading carpool {agency_id}:{carpool_id}.") + continue + + existing_enhanced_carpool = _load_carpool_if_exists(agency_id, carpool.id, folder='data/enhanced') + if not existing_enhanced_carpool or existing_enhanced_carpool.lastUpdated != carpool.lastUpdated: logger.info(f"Enhancing restored trip {carpool_file_name}") try: - with open(carpool_file_name) as carpool_file: - carpool = Carpool(**(json.load(carpool_file))) - enhance_trip(carpool) + enhance_trip(carpool) except Exception as e: logger.warning("Issue during restore of carpool %s: %s", carpool_file_name, repr(e)) +def carpool_exists(agency_id: str, carpool_id: str, folder: str ='data/enhanced'): + return os.path.exists(f"{folder}/{agency_id}/{carpool_id}.json") + +def _load_carpool_if_exists(agency_id: str, carpool_id: str, folder: str ='data/enhanced'): + if carpool_exists(agency_id, carpool_id, folder): + try: + return _load_carpool_from_path(f"{folder}/{agency_id}/{carpool_id}.json") + except Exception as e: + # An error on restore could be caused by model changes, + # in such a case, it need's to be recreated + logger.warning(f"Could not restore trip {folder}/{agency_id}/{carpool_id}.json: {repr(e)}") + + return None + @router.post("/", operation_id="addcarpool", @@ -156,13 +172,15 @@ async def set_lastUpdated_if_unset(carpool): carpool.lastUpdated = datetime.now() -async def load_carpool(agency_id, carpool_id) -> Carpool: - with open(f'data/carpool/{agency_id}/{carpool_id}.json', 'r', encoding='utf-8') as f: +async def load_carpool(agency_id, carpool_id, folder: str ='data/carpool') -> Carpool: + return _load_carpool_from_path(f'{folder}/{agency_id}/{carpool_id}.json') + +def _load_carpool_from_path(path: str): + with open(path, 'r', encoding='utf-8') as f: dict = json.load(f) carpool = Carpool(**dict) return carpool - async def save_carpool(carpool, folder: str = 'data/carpool'): with open(f'{folder}/{carpool.agency}/{carpool.id}.json', 'w', encoding='utf-8') as f: f.write(carpool.json()) From 8f6d813e5e0f062a7dd8b3c702e7915606ec4523 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Wed, 5 Mar 2025 11:35:42 +0100 Subject: [PATCH 47/50] Fixed carpool example coordinates --- amarillo/models/Carpool.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/amarillo/models/Carpool.py b/amarillo/models/Carpool.py index b9c019c..60bfd6d 100644 --- a/amarillo/models/Carpool.py +++ b/amarillo/models/Carpool.py @@ -391,18 +391,20 @@ class Carpool(BaseModel): "agency": "mfdz", "deeplink": "http://mfdz.de", "stops": [ - { - "id": "de:12073:900340137::2", "name": "ABC", - "lat": 45, "lon": 9 - }, - { - "id": "de:12073:900340137::3", "name": "XYZ", - "lat": 45, "lon": 9 - } + { + "name": "Stuttgart", + "lat": 48.783138, + "lon": 9.181288 + }, + { + "name": "Ulm", + "lat": 48.39892, + "lon": 9.98419 + } ], "departureTime": "12:34", - "departureDate": "2022-03-30", - "lastUpdated": "2022-03-30T12:34:00+00:00" - } + "departureDate": "2025-05-01", + "lastUpdated": "2025-02-18T12:34:00+00:00" + } """ }) From f887fb25424ec8584db119ce0c0792c686b96fee Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Wed, 5 Mar 2025 11:38:02 +0100 Subject: [PATCH 48/50] Save failed carpool enhancements to data/failed --- amarillo/routers/carpool.py | 12 +++++++++++- amarillo/utils/utils.py | 6 ++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/amarillo/routers/carpool.py b/amarillo/routers/carpool.py index a896ea0..5548947 100644 --- a/amarillo/routers/carpool.py +++ b/amarillo/routers/carpool.py @@ -41,9 +41,19 @@ def enhance_trip(carpool: Carpool): with open(filename, 'w', encoding='utf-8') as f: f.write(enhanced_carpool.model_dump_json()) except ConnectionError: + handle_failed_carpool_enhancement(carpool) logger.error("Could not connect to enhancer: make sure amarillo-enhancer is running and your ENHANCER_URL environment variable is configured correctly") + except requests.HTTPError as e: + handle_failed_carpool_enhancement(carpool) + logger.error(f"Error enhancing trip '{carpool.agency}:{carpool.id}': {e.response.status_code} {e.response.json()}") except Exception as e: - logger.error(f"Error enhancing trip '{carpool.id}': {e}") + handle_failed_carpool_enhancement(carpool) + logger.error(f"Error enhancing trip '{carpool.agency}:{carpool.id}': {e}") + +def handle_failed_carpool_enhancement(carpool: Carpool): + assert_folder_exists(f'data/failed/{carpool.agency}/') + with open(f'data/failed/{carpool.agency}/{carpool.id}.json', 'w', encoding='utf-8') as f: + f.write(carpool.model_dump_json()) def enhance_missing_carpools(): logger.info(f"Enhancing missing restored trips...") diff --git a/amarillo/utils/utils.py b/amarillo/utils/utils.py index 9d060b3..93f7b61 100644 --- a/amarillo/utils/utils.py +++ b/amarillo/utils/utils.py @@ -47,6 +47,12 @@ def geodesic_distance_in_m(coord1, coord2): lats = [coord1[1], coord2[1]] return geod.line_lengths(lons, lats)[0] +def carpool_distance_in_m(carpool): + if len(carpool.stops) < 2: + return 0 + s1 = carpool.stops[0] + s2 = carpool.stops[-1] + return geodesic_distance_in_m((s1.lon, s1.lat),(s2.lon, s2.lat)) def copy_static_files(files_and_dirs_to_copy): amarillo_dir = Path(__file__).parents[1] From 7956131f36af4e91582a4b77216ddf3eefee8f96 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Wed, 5 Mar 2025 12:41:42 +0100 Subject: [PATCH 49/50] Removed unused dependencies --- Dockerfile | 11 ----------- pyproject.toml | 13 +++++++------ requirements.txt | 14 ++++---------- 3 files changed, 11 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6a0bc14..2cb8f63 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,17 +4,6 @@ LABEL maintainer="info@mfdz.de" WORKDIR /app -RUN \ - apt update \ - && apt install -y \ - # GDAL headers are required for fiona, which is required for geopandas. - # Also gcc is used to compile C++ code. - libgdal-dev g++ \ - # libspatialindex is required for rtree. - libspatialindex-dev \ - # Remove package index obtained by `apt update`. - && rm -rf /var/lib/apt/lists/* - ENV ADMIN_TOKEN='' ENV RIDE2GO_TOKEN='' diff --git a/pyproject.toml b/pyproject.toml index b284faa..9fa511a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,21 +1,22 @@ [project] name = "amarillo" -version = "2.0.0" +version = "2.0.1" description = "Aggregates and enhances carpooling-offers and publishes them as GTFS(-RT)" readme = "README.md" license = {file = "LICENSE"} keywords = ["amarillo", "ridesharing", "carpooling", "gtfs", "gtfs-rt"] dependencies = [ - "fastapi[all]==0.109.0", - "geopandas==0.14", - "uvicorn[standard]==0.23.2", + "fastapi[all]==0.111.0", + "uvicorn[standard]~=0.29.0", "pydantic[dotenv]==2.4.2", - "protobuf==3.20.3", + "rtree==1.1.0", + "setproctitle==1.3.3", "starlette~=0.35", "requests==2.31.0", "pyproj==3.6.1", "geojson-pydantic==1.0.1", - "watchdog==3.0.0", + "idna>=3.7", + "numpy==1.26.4", ] [tool.setuptools.packages] diff --git a/requirements.txt b/requirements.txt index 292a8f4..4314b77 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,11 @@ -amarillo-metrics==0.0.6 -fastapi[all]==0.109.0 -geopandas==0.14 -uvicorn[standard]==0.23.2 +fastapi[all]==0.111.0 +uvicorn[standard]~=0.29.0 pydantic[dotenv]==2.4.2 -protobuf==3.20.3 rtree==1.1.0 -schedule==1.2.1 setproctitle==1.3.3 starlette~=0.35 -pandas==2.1.1 requests==2.31.0 -Shapely==2.0.2 -pygeos==0.14 pyproj==3.6.1 geojson-pydantic==1.0.1 -watchdog==3.0.0 \ No newline at end of file +idna>=3.7 +numpy==1.26.4 \ No newline at end of file From bbf9fd735b9c4de0dd9355eebc99576e55bb8b42 Mon Sep 17 00:00:00 2001 From: Francia Csaba Date: Wed, 5 Mar 2025 12:43:37 +0100 Subject: [PATCH 50/50] Update README.md --- README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c0c5990..a2dd39f 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,17 @@ An Amarillo is a [yellow-dressed person](https://www.dreamstime.com/sancti-spiri ## Setup -- Python 3.9.2 with pip +- Python 3.10 with pip - python3-venv -Create a virtual environment `python3 -m venv venv`. +Create a virtual environment: +`python3 -m venv venv`. + +Activate the environment: +`. venv/bin/activate` + +Install the dependencies: `pip install -r requirements.txt`. -Activate the environment and install the dependencies `pip install -r requirements.txt`. Run `uvicorn amarillo.main:app`. @@ -66,5 +71,5 @@ In the top directory, run `pytest amarillo/tests`. Based on [tiangolo/uvicorn-gunicorn:python3.10-slim](https://github.com/tiangolo/uvicorn-gunicorn-docker) -- build `docker build -t amarillo --build-arg="PLUGINS=amarillo-metrics" .` +- build `docker build -t amarillo .` - run `docker run --rm --name amarillo -p 8000:80 -e MAX_WORKERS="1" -e ADMIN_TOKEN=$ADMIN_TOKEN -e RIDE2GO_TOKEN=$RIDE2GO_TOKEN -e TZ=Europe/Berlin -v $(pwd)/data:/app/data amarillo`