diff options
| -rw-r--r-- | fietsboek/config.py | 17 | ||||
| -rw-r--r-- | fietsboek/hittekaart.py | 61 | ||||
| -rw-r--r-- | fietsboek/scripts/fietscron.py | 5 | ||||
| -rw-r--r-- | fietsboek/scripts/fietsctl.py | 5 | ||||
| -rw-r--r-- | poetry.lock | 23 | ||||
| -rw-r--r-- | pyproject.toml | 5 | ||||
| -rw-r--r-- | tox.ini | 2 |
7 files changed, 79 insertions, 39 deletions
diff --git a/fietsboek/config.py b/fietsboek/config.py index beea820..bd2347a 100644 --- a/fietsboek/config.py +++ b/fietsboek/config.py @@ -64,6 +64,11 @@ KNOWN_TILE_LAYERS = [ "hiking", ] +WARNINGS = { + "hittekaart.bin": "hittekaart is now used via a Python module. " + "Enable extra `hittekaart` to install the dependency.", +} + class ValidationError(Exception): """Exception for malformed configurations. @@ -203,9 +208,6 @@ class Config(BaseModel): tile_layers: list[TileLayerConfig] = [] """Tile layers.""" - hittekaart_bin: str = Field("", alias="hittekaart.bin") - """Path to the hittekaart binary.""" - hittekaart_autogenerate: PyramidList = Field([], alias="hittekaart.autogenerate") """Overlay maps to automatically generate.""" @@ -347,10 +349,17 @@ def parse(config: dict) -> Config: keys.discard(_field_name(field_name, field)) keys -= KNOWN_PYRAMID_SETTINGS + _warn_unknown(keys) + + return parsed_config + + +def _warn_unknown(keys): for key in keys: LOGGER.warning("Unknown configuration key: %r", key) - return parsed_config + if warning := WARNINGS.get(key): + LOGGER.warning(warning) def _field_name(field_name, field): diff --git a/fietsboek/hittekaart.py b/fietsboek/hittekaart.py index 15f2855..3b0c103 100644 --- a/fietsboek/hittekaart.py +++ b/fietsboek/hittekaart.py @@ -6,11 +6,13 @@ import enum import logging import shutil -import subprocess import tempfile from pathlib import Path -from typing import Optional +try: + import hittekaart_py +except ImportError: + pass from sqlalchemy import select from sqlalchemy.orm import aliased from sqlalchemy.orm.session import Session @@ -21,6 +23,13 @@ from .models.track import TrackType LOGGER = logging.getLogger(__name__) +COMPRESSION_MAP = { + ".br": "brotli", + ".gz": "gzip", +} + +TILEHUNTER_ZOOM = 14 + class Mode(enum.Enum): """Heatmap generation mode. @@ -38,7 +47,6 @@ def generate( mode: Mode, input_files: list[Path], *, - exe_path: Optional[Path] = None, threads: int = 0, ): """Calls hittekaart with the given arguments. @@ -47,13 +55,29 @@ def generate( sqlite output mode. :param mode: What to generate. :param input_files: List of paths to the input files. - :param exe_path: Path to the hittekaart binary. If not given, - ``hittekaart`` is searched in the path. :param threads: Number of threads that ``hittekaart`` should use. Defaults to 0, which uses all available cores. """ + try: + hittekaart_py + except NameError: + raise RuntimeError("hittekaart not available") from None + if not input_files: return + + renderer: hittekaart_py.HeatmapRenderer | hittekaart_py.TilehuntRenderer + if mode == Mode.HEATMAP: + renderer = hittekaart_py.HeatmapRenderer() + elif mode == Mode.TILEHUNTER: + renderer = hittekaart_py.TilehuntRenderer(TILEHUNTER_ZOOM) + + LOGGER.debug("Loading tracks ...") + tracks = [ + hittekaart_py.Track.from_file(bytes(input_file), COMPRESSION_MAP.get(input_file.suffix)) + for input_file in input_files + ] + LOGGER.debug("Tracks loaded!") # There are two reasons why we do the tempfile dance: # 1. hittekaart refuses to overwrite existing files # 2. This way we can (hope for?) an atomic move (at least if temporary file @@ -61,23 +85,12 @@ def generate( # this, but for now, it's alright. with tempfile.TemporaryDirectory() as tempdir: tmpfile = Path(tempdir) / "hittekaart.sqlite" - binary = str(exe_path) if exe_path else "hittekaart" - cmdline = [ - binary, - "--sqlite", - "-o", - str(tmpfile), - "-m", - mode.value, - "-t", - str(threads), - "--", - ] - cmdline.extend(map(str, input_files)) - LOGGER.debug("Running %r", cmdline) - subprocess.run(cmdline, check=True, stdout=subprocess.DEVNULL) - - LOGGER.debug("Moving temporary file") + sink = hittekaart_py.Storage.Sqlite(bytes(tmpfile)) + settings = hittekaart_py.Settings(threads=threads) + LOGGER.debug("Running hittekaart (renderer %r) to %r", renderer, tmpfile) + hittekaart_py.generate(settings, tracks, renderer, sink) + + LOGGER.debug("Moving temporary file %r to %r", tmpfile, output) shutil.move(tmpfile, output) @@ -87,7 +100,6 @@ def generate_for( data_manager: DataManager, mode: Mode, *, - exe_path: Optional[Path] = None, threads: int = 0, ): """Uses :meth:`generate` to generate a heatmap for the given user. @@ -104,7 +116,6 @@ def generate_for( :param dbsession: The database session. :param data_manager: The data manager. :param mode: The mode of the heatmap. - :param exe_path: See :meth:`generate`. :param threads: See :meth:`generate`. """ # pylint: disable=too-many-arguments @@ -133,7 +144,7 @@ def generate_for( Mode.TILEHUNTER: user_dir.tilehunt_path(), } - generate(output_paths[mode], mode, input_paths, exe_path=exe_path, threads=threads) + generate(output_paths[mode], mode, input_paths, threads=threads) __all__ = ["Mode", "generate", "generate_for"] diff --git a/fietsboek/scripts/fietscron.py b/fietsboek/scripts/fietscron.py index a3f3f54..1a8e855 100644 --- a/fietsboek/scripts/fietscron.py +++ b/fietsboek/scripts/fietscron.py @@ -3,7 +3,6 @@ import datetime import logging import logging.config -from pathlib import Path import click import gpxpy @@ -142,7 +141,6 @@ def run_hittekaart(engine: Engine, data_manager: DataManager, redis: Redis, conf # re-generate all maps over time (e.g. if the hittekaart version changes or # we miss an update). modes = [hittekaart.Mode(mode) for mode in config.hittekaart_autogenerate] - exe_path = Path(config.hittekaart_bin) if config.hittekaart_bin else None session = Session(engine) had_hq_item = False @@ -164,7 +162,6 @@ def run_hittekaart(engine: Engine, data_manager: DataManager, redis: Redis, conf session, data_manager, mode, - exe_path=exe_path, threads=config.hittekaart_threads, ) @@ -188,7 +185,7 @@ def run_hittekaart(engine: Engine, data_manager: DataManager, redis: Redis, conf for mode in modes: LOGGER.info("Generating %s for user %d (low-priority)", mode.value, user.id) hittekaart.generate_for( - user, session, data_manager, mode, exe_path=exe_path, threads=config.hittekaart_threads + user, session, data_manager, mode, threads=config.hittekaart_threads ) diff --git a/fietsboek/scripts/fietsctl.py b/fietsboek/scripts/fietsctl.py index ad06e91..630660f 100644 --- a/fietsboek/scripts/fietsctl.py +++ b/fietsboek/scripts/fietsctl.py @@ -313,7 +313,6 @@ def cmd_user_hittekaart( else: query = models.User.query_by_email(email) - exe_path = env["request"].config.hittekaart_bin threads = env["request"].config.hittekaart_threads with env["request"].tm: dbsession = env["request"].dbsession @@ -337,9 +336,7 @@ def cmd_user_hittekaart( click.echo(f"Generating overlay maps for {click.style(user.name, fg=FG_USER_NAME)}...") for mode in modes: - hittekaart.generate_for( - user, dbsession, data_manager, mode, exe_path=exe_path, threads=threads - ) + hittekaart.generate_for(user, dbsession, data_manager, mode, threads=threads) click.echo(f"Generated {mode.value}") diff --git a/poetry.lock b/poetry.lock index 9bf75d6..3763009 100644 --- a/poetry.lock +++ b/poetry.lock @@ -843,6 +843,24 @@ docs = ["Sphinx", "furo"] test = ["objgraph", "psutil", "setuptools"] [[package]] +name = "hittekaart-py" +version = "0.1.0" +description = "" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"hittekaart\"" +files = [] +develop = false + +[package.source] +type = "git" +url = "https://gitlab.com/dunj3/hittekaart" +reference = "013dc01683c42177e132847475c8b57d1a67fc14" +resolved_reference = "013dc01683c42177e132847475c8b57d1a67fc14" +subdirectory = "hittekaart-py" + +[[package]] name = "hupper" version = "1.12.1" description = "Integrated process monitor for developing and reloading daemons." @@ -2653,7 +2671,10 @@ transaction = ">=1.6.0" [package.extras] test = ["zope.testing"] +[extras] +hittekaart = ["hittekaart-py"] + [metadata] lock-version = "2.1" python-versions = ">=3.11" -content-hash = "7d457584ce55a0ded1aa21bdc7cf0cca26e4e30915e762d09d5d2d7f0ba4752f" +content-hash = "b2bd0fd69ceab70c448a4e4b517ce30c65c17148f0ac9698eaadd4673b66f36a" diff --git a/pyproject.toml b/pyproject.toml index 8b01a42..cea085f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,11 @@ documentation = "https://docs.fietsboek.org/" homepage = "https://fietsboek.org/" repository = "https://gitlab.com/dunj3/fietsboek" +[project.optional-dependencies] +hittekaart = [ + "hittekaart-py @ git+https://gitlab.com/dunj3/hittekaart@013dc01683c42177e132847475c8b57d1a67fc14#subdirectory=hittekaart-py", +] + [tool.poetry] classifiers = [ 'Development Status :: 3 - Alpha', @@ -60,7 +60,7 @@ commands = [testenv:mypy] commands_pre = - poetry install --with types + poetry install --with types --extras hittekaart commands = mypy fietsboek |