diff options
| -rw-r--r-- | fietsboek/config.py | 16 | ||||
| -rw-r--r-- | fietsboek/scripts/fietscron.py | 70 |
2 files changed, 85 insertions, 1 deletions
diff --git a/fietsboek/config.py b/fietsboek/config.py index 3fc191f..42d717c 100644 --- a/fietsboek/config.py +++ b/fietsboek/config.py @@ -198,6 +198,9 @@ class Config(BaseModel): hittekaart_bin: str = Field("", alias="hittekaart.bin") """Path to the hittekaart binary.""" + hittekaart_autogenerate: PyramidList = Field([], alias="hittekaart.autogenerate") + """Overlay maps to automatically generate.""" + @validator("session_key") def _good_session_key(cls, value): """Ensures that the session key has been changed from its default @@ -225,6 +228,19 @@ class Config(BaseModel): raise ValueError("Unknown stamen maps: " + ", ".join(bad_maps)) return value + @validator("hittekaart_autogenerate") + def _known_hittekaart_modes(cls, value): + """Ensures that the hittekaart modes are valid.""" + # pylint: disable=import-outside-toplevel + from . import hittekaart + + modes = set(value) + known_modes = {mode.value for mode in hittekaart.Mode} + bad_modes = modes - known_modes + if bad_modes: + raise ValueError("Unknown hittekaart overlays: " + ", ".join(bad_modes)) + return value + def derive_secret(self, what_for): """Derive a secret for other parts of the application. diff --git a/fietsboek/scripts/fietscron.py b/fietsboek/scripts/fietscron.py index a142f39..e422fb2 100644 --- a/fietsboek/scripts/fietscron.py +++ b/fietsboek/scripts/fietscron.py @@ -2,15 +2,19 @@ import datetime import logging import logging.config +from pathlib import Path import click import pyramid.paster +import redis as mod_redis +from redis import Redis from sqlalchemy import create_engine, delete, exists, not_, select from sqlalchemy.engine import Engine from sqlalchemy.orm import Session from .. import config as mod_config -from .. import models +from .. import hittekaart, models +from ..config import Config from ..data import DataManager LOGGER = logging.getLogger(__name__) @@ -32,6 +36,7 @@ def cli(config): \b * Deletes pending uploads that are older than 24 hours. * Rebuilds the cache for missing tracks. + * (optional) Runs ``hittekaart`` to generate heatmaps """ logging.config.fileConfig(config) settings = pyramid.paster.get_appsettings(config) @@ -50,6 +55,10 @@ def cli(config): remove_old_uploads(engine) rebuild_cache(engine, data_manager) + if config.hittekaart_autogenerate: + redis = mod_redis.from_url(config.redis_url) + run_hittekaart(engine, data_manager, redis, config) + def remove_old_uploads(engine: Engine): """Removes old uploads from the database.""" @@ -77,4 +86,63 @@ def rebuild_cache(engine: Engine, data_manager: DataManager): session.commit() +def run_hittekaart(engine: Engine, data_manager: DataManager, redis: Redis, config: Config): + """Run outstanding hittekaart requests.""" + # The logic here is as follows: + # We keep two lists: a high-priority one and a low-priority one + # If there are high priority entries, we run all of them. + # They are refilled when users upload tracks. + # If there are no high priority entries, we run a single low priority one. + # If there are no low priority entries, we re-fill the queue by adding all tracks. + # This way, we ensure that we "catch" modifications fast, but we also + # re-generate all maps over time (e.g. if the hittekaart version changes or + # we miss an update). + modes = [hittekaart.Mode(mode) for mode in config.hittekaart_autogenerate] + exe_path = Path(config.hittekaart_bin) if config.hittekaart_bin else None + session = Session(engine) + had_hq_item = False + + while True: + # High-priority queue + item = redis.spop("hittekaart:queue:high") + if item is None: + break + user = session.execute(select(models.User).filter_by(id=int(item))).scalar() + if user is None: + LOGGER.debug("User %d had a queue entry but was not found", item) + break + + for mode in modes: + LOGGER.info("Generating %s for user %d", mode.value, user.id) + hittekaart.generate_for(user, session, data_manager, mode, exe_path=exe_path) + + if had_hq_item: + return + + # Low-priority queue + item = redis.spop("hittekaart:queue:low") + if item is None: + refill_queue(session, redis) + item = redis.spop("hittekaart:queue:low") + if item is None: + LOGGER.debug("No users, no hittekaarts") + return + + user = session.execute(select(models.User).filter_by(id=int(item))).scalar() + if user is None: + LOGGER.debug("User %d had a queue entry but was not found", item) + return + + for mode in modes: + LOGGER.info("Generating %s for user %d", mode.value, user.id) + hittekaart.generate_for(user, session, data_manager, mode, exe_path=exe_path) + + +def refill_queue(session: Session, redis: Redis): + """Refills the low-priority hittekaart queue by adding all users to it.""" + LOGGER.debug("Refilling low-priority queue") + for user in session.execute(select(models.User)).scalars(): + redis.sadd("hittekaart:queue:low", str(user.id)) + + __all__ = ["cli"] |