make fietscron call hittekaart

2 files changed, 85 insertions, 1 deletions

diff --git a/fietsboek/config.py b/fietsboek/config.py
index 3fc191f..42d717c 100644
--- a/fietsboek/config.py
+++ b/fietsboek/config.py
@@ -198,6 +198,9 @@ class Config(BaseModel):
     hittekaart_bin: str = Field("", alias="hittekaart.bin")
     """Path to the hittekaart binary."""
 
+    hittekaart_autogenerate: PyramidList = Field([], alias="hittekaart.autogenerate")
+    """Overlay maps to automatically generate."""
+
     @validator("session_key")
     def _good_session_key(cls, value):
         """Ensures that the session key has been changed from its default
@@ -225,6 +228,19 @@ class Config(BaseModel):
             raise ValueError("Unknown stamen maps: " + ", ".join(bad_maps))
         return value
 
+    @validator("hittekaart_autogenerate")
+    def _known_hittekaart_modes(cls, value):
+        """Ensures that the hittekaart modes are valid."""
+        # pylint: disable=import-outside-toplevel
+        from . import hittekaart
+
+        modes = set(value)
+        known_modes = {mode.value for mode in hittekaart.Mode}
+        bad_modes = modes - known_modes
+        if bad_modes:
+            raise ValueError("Unknown hittekaart overlays: " + ", ".join(bad_modes))
+        return value
+
     def derive_secret(self, what_for):
         """Derive a secret for other parts of the application.
 
diff --git a/fietsboek/scripts/fietscron.py b/fietsboek/scripts/fietscron.py
index a142f39..e422fb2 100644
--- a/fietsboek/scripts/fietscron.py
+++ b/fietsboek/scripts/fietscron.py
@@ -2,15 +2,19 @@
 import datetime
 import logging
 import logging.config
+from pathlib import Path
 
 import click
 import pyramid.paster
+import redis as mod_redis
+from redis import Redis
 from sqlalchemy import create_engine, delete, exists, not_, select
 from sqlalchemy.engine import Engine
 from sqlalchemy.orm import Session
 
 from .. import config as mod_config
-from .. import models
+from .. import hittekaart, models
+from ..config import Config
 from ..data import DataManager
 
 LOGGER = logging.getLogger(__name__)
@@ -32,6 +36,7 @@ def cli(config):
     \b
     * Deletes pending uploads that are older than 24 hours.
     * Rebuilds the cache for missing tracks.
+    * (optional) Runs ``hittekaart`` to generate heatmaps
     """
     logging.config.fileConfig(config)
     settings = pyramid.paster.get_appsettings(config)
@@ -50,6 +55,10 @@ def cli(config):
     remove_old_uploads(engine)
     rebuild_cache(engine, data_manager)
 
+    if config.hittekaart_autogenerate:
+        redis = mod_redis.from_url(config.redis_url)
+        run_hittekaart(engine, data_manager, redis, config)
+
 
 def remove_old_uploads(engine: Engine):
     """Removes old uploads from the database."""
@@ -77,4 +86,63 @@ def rebuild_cache(engine: Engine, data_manager: DataManager):
         session.commit()
 
 
+def run_hittekaart(engine: Engine, data_manager: DataManager, redis: Redis, config: Config):
+    """Run outstanding hittekaart requests."""
+    # The logic here is as follows:
+    # We keep two lists: a high-priority one and a low-priority one
+    # If there are high priority entries, we run all of them.
+    # They are refilled when users upload tracks.
+    # If there are no high priority entries, we run a single low priority one.
+    # If there are no low priority entries, we re-fill the queue by adding all tracks.
+    # This way, we ensure that we "catch" modifications fast, but we also
+    # re-generate all maps over time (e.g. if the hittekaart version changes or
+    # we miss an update).
+    modes = [hittekaart.Mode(mode) for mode in config.hittekaart_autogenerate]
+    exe_path = Path(config.hittekaart_bin) if config.hittekaart_bin else None
+    session = Session(engine)
+    had_hq_item = False
+
+    while True:
+        # High-priority queue
+        item = redis.spop("hittekaart:queue:high")
+        if item is None:
+            break
+        user = session.execute(select(models.User).filter_by(id=int(item))).scalar()
+        if user is None:
+            LOGGER.debug("User %d had a queue entry but was not found", item)
+            break
+
+        for mode in modes:
+            LOGGER.info("Generating %s for user %d", mode.value, user.id)
+            hittekaart.generate_for(user, session, data_manager, mode, exe_path=exe_path)
+
+    if had_hq_item:
+        return
+
+    # Low-priority queue
+    item = redis.spop("hittekaart:queue:low")
+    if item is None:
+        refill_queue(session, redis)
+        item = redis.spop("hittekaart:queue:low")
+        if item is None:
+            LOGGER.debug("No users, no hittekaarts")
+            return
+
+    user = session.execute(select(models.User).filter_by(id=int(item))).scalar()
+    if user is None:
+        LOGGER.debug("User %d had a queue entry but was not found", item)
+        return
+
+    for mode in modes:
+        LOGGER.info("Generating %s for user %d", mode.value, user.id)
+        hittekaart.generate_for(user, session, data_manager, mode, exe_path=exe_path)
+
+
+def refill_queue(session: Session, redis: Redis):
+    """Refills the low-priority hittekaart queue by adding all users to it."""
+    LOGGER.debug("Refilling low-priority queue")
+    for user in session.execute(select(models.User)).scalars():
+        redis.sadd("hittekaart:queue:low", str(user.id))
+
+
 __all__ = ["cli"]