From aebb3cb0bf39f47bd3c1cf44a2a3605405a2de59 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Fri, 3 Feb 2023 22:44:49 +0100 Subject: initial work on transformers So far it doesn't really do much yet, but it does have the machinery to list the available transformers and run them. It also memorizes if the transformers even need to be run at all, to save time if the current configuration already matches. The parameter UI still needs some work. This is fine because the first transformer will not have any parameters (it's just the elevation fix). We probably don't want to have a method that returns Markup, as that makes it hard to implement localization in there, and the method would need to be aware of bootstrap. Another point to think about is documentation. I'd like some information for the user what the "transformers" are, so we'll probably add a small tagline and later extend the documentation with some more information (I want a user chapter in there at some point anyway). --- fietsboek/__init__.py | 2 + fietsboek/actions.py | 63 ++++++++- .../alembic/versions/20230203_3149aa2d0114.py | 22 ++++ fietsboek/models/track.py | 19 ++- fietsboek/templates/edit_form.jinja2 | 34 +++++ fietsboek/transformers/__init__.py | 145 +++++++++++++++++++++ fietsboek/views/edit.py | 1 + 7 files changed, 282 insertions(+), 4 deletions(-) create mode 100644 fietsboek/alembic/versions/20230203_3149aa2d0114.py create mode 100644 fietsboek/transformers/__init__.py diff --git a/fietsboek/__init__.py b/fietsboek/__init__.py index 9112576..95ff394 100644 --- a/fietsboek/__init__.py +++ b/fietsboek/__init__.py @@ -29,6 +29,7 @@ from pyramid.session import SignedCookieSessionFactory from . import config as mod_config from . import jinja2 as mod_jinja2 +from . import transformers from .data import DataManager from .pages import Pages from .security import SecurityPolicy @@ -138,5 +139,6 @@ def main(_global_config, **settings): jinja2_env.filters["format_datetime"] = mod_jinja2.filter_format_datetime jinja2_env.filters["local_datetime"] = mod_jinja2.filter_local_datetime jinja2_env.globals["embed_tile_layers"] = mod_jinja2.global_embed_tile_layers + jinja2_env.globals["list_transformers"] = transformers.list_transformers return config.make_wsgi_app() diff --git a/fietsboek/actions.py b/fietsboek/actions.py index f1a32fc..350faaf 100644 --- a/fietsboek/actions.py +++ b/fietsboek/actions.py @@ -10,13 +10,15 @@ import logging import re from typing import List +import brotli +import gpxpy from pyramid.request import Request from sqlalchemy import select from sqlalchemy.orm.session import Session -from fietsboek import models, util -from fietsboek.data import DataManager -from fietsboek.models.track import TrackType, Visibility +from . import models, transformers, util +from .data import DataManager +from .models.track import TrackType, Visibility LOGGER = logging.getLogger(__name__) @@ -154,3 +156,58 @@ def edit_images(request: Request, track: models.Track): image_meta = models.ImageMetadata.get_or_create(request.dbsession, track, image_id) image_meta.description = description request.dbsession.add(image_meta) + + +def execute_transformers(request: Request, track: models.Track): + """Execute the transformers for the given track. + + Note that this function "short circuits" if the saved transformer settings + already match the settings given in the request. + + :param request: The request. + :param track: The track. + """ + # pylint: disable=too-many-locals + LOGGER.debug("Executing transformers for %d", track.id) + + settings = [] + for tfm in transformers.list_transformers(): + ident = tfm.identifier() + prefix = f"transformer[{ident}]" + req_params = {} + for name, val in request.params.items(): + if name.startswith(prefix): + name = name[len(prefix) :] + name = name.strip("[]") + req_params[name] = val + + if req_params.get("") == "on": + params = tfm().parameters + params.read_from_request(req_params) + settings.append((ident, params)) + + serialized = [[tfm_id, params.dict()] for tfm_id, params in settings] + if serialized == track.transformers: + LOGGER.debug("Applied transformations mach on %d, skipping", track.id) + return + + # We always start with the backup, that way we don't get "deepfried GPX" + # files by having the same filters run multiple times on the same input. + # They are not idempotent after all. + manager = request.data_manager.open(track.id) + gpx_bytes = manager.backup_path().read_bytes() + gpx_bytes = brotli.decompress(gpx_bytes) + gpx = gpxpy.parse(gpx_bytes) + + tfms = {tfm.identifier(): tfm for tfm in transformers.list_transformers()} + for tfm_id, params in settings: + transformer = tfms[tfm_id]() + transformer.parameters = params + LOGGER.debug("Running %s with %r", transformer, params) + transformer.execute(gpx) + + LOGGER.debug("Saving transformed file for %d", track.id) + manager.compress_gpx(gpx.to_xml().encode("utf-8")) + + LOGGER.debug("Saving new transformers on %d", track.id) + track.transformers = serialized diff --git a/fietsboek/alembic/versions/20230203_3149aa2d0114.py b/fietsboek/alembic/versions/20230203_3149aa2d0114.py new file mode 100644 index 0000000..eb9ef78 --- /dev/null +++ b/fietsboek/alembic/versions/20230203_3149aa2d0114.py @@ -0,0 +1,22 @@ +"""add transformer column + +Revision ID: 3149aa2d0114 +Revises: c939800af428 +Create Date: 2023-02-03 21:44:39.429564 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = '3149aa2d0114' +down_revision = 'c939800af428' +branch_labels = None +depends_on = None + +def upgrade(): + op.add_column('tracks', sa.Column('transformers', sa.JSON(), nullable=True)) + op.execute('UPDATE tracks SET transformers="[]";') + +def downgrade(): + op.drop_column('tracks', 'transformers') diff --git a/fietsboek/models/track.py b/fietsboek/models/track.py index 9f9d7f6..a7217bf 100644 --- a/fietsboek/models/track.py +++ b/fietsboek/models/track.py @@ -32,6 +32,7 @@ from pyramid.httpexceptions import HTTPNotFound from pyramid.i18n import Localizer from pyramid.i18n import TranslationString as _ from sqlalchemy import ( + JSON, Column, DateTime, Enum, @@ -174,6 +175,8 @@ class Track(Base): :vartype link_secret: str :ivar type: Type of the track :vartype type: TrackType + :ivar transformers: The enabled transformers together with their parameters. + :vartype transformers: list[tuple[str, dict]] :ivar owner: Owner of the track. :vartype owner: fietsboek.models.user.User :ivar cache: Cache for the computed track metadata. @@ -198,6 +201,7 @@ class Track(Base): visibility = Column(Enum(Visibility)) link_secret = Column(Text) type = Column(Enum(TrackType)) + transformers = Column(JSON) owner = relationship("User", back_populates="tracks") cache = relationship( @@ -397,12 +401,25 @@ class Track(Base): self.tags.append(Tag(tag=to_add)) to_delete = [] - for (i, tag) in enumerate(self.tags): + for i, tag in enumerate(self.tags): if tag.tag.lower() not in lower_tags: to_delete.append(i) for i in to_delete[::-1]: del self.tags[i] + def transformer_params_for(self, transformer_id: str) -> Optional[dict]: + """Returns the transformer parameters for the given transformer. + + If the transformer is not active, returns ``None``. + + :param transformer_id: The string ID of the transformer. + :return: The settings as a dictionary. + """ + for t_id, settings in self.transformers: + if t_id == transformer_id: + return settings + return None + class TrackWithMetadata: """A class to add metadata to a :class:`Track`. diff --git a/fietsboek/templates/edit_form.jinja2 b/fietsboek/templates/edit_form.jinja2 index bfd45a1..67c01c9 100644 --- a/fietsboek/templates/edit_form.jinja2 +++ b/fietsboek/templates/edit_form.jinja2 @@ -142,4 +142,38 @@ + +
+
+ {% for transformer in list_transformers() %} +
+

+ +

+
+
+
+ {% set params = track.transformer_params_for(transformer.identifier()) %} + + +
+ + +
+ + + {% if params is not none %} + {{ transformer.parameter_model().parse_obj(params).html_ui("transformer[{}][%%]".format(transformer.identifier())) }} + {% else %} + {{ transformer().parameters.html_ui("transformer[{}][%%]".format(transformer.identifier())) }} + {% endif %} +
+
+ {% endfor %} +
+
{% endmacro %} diff --git a/fietsboek/transformers/__init__.py b/fietsboek/transformers/__init__.py new file mode 100644 index 0000000..b4ed532 --- /dev/null +++ b/fietsboek/transformers/__init__.py @@ -0,0 +1,145 @@ +"""Fietsboek GPX transformers. + +A "transformer" is something like a filter - it takes in a GPX file and applies +some small corrections, such as smoothing out the elevation. In order to avoid +confusion with the naming (and the "filters" you have when searching for +tracks), we call those "GPX filters" *transformers*. + +This module defines the abstract interface for transformers, as well as +function to load and apply transformers. +""" + +from abc import ABC, abstractmethod +from collections.abc import Mapping + +from gpxpy.gpx import GPX +from markupsafe import Markup +from pydantic import BaseModel +from pyramid.i18n import TranslationString + +_ = TranslationString + + +class Parameters(BaseModel): + """Parameters for a transformer. + + This is basically a wrapper around pydantic models that allows the + parameters to be serialized from and to POST request parameters. + """ + + def html_ui(self, name_template: str) -> Markup: + """Renders a HTML UI for this parameter set. + + :param name_template: The template for the HTML form element names, + with a %% placeholder for the parameter name. + :return: The rendered UI, ready for inclusion. + """ + # TODO: Implement this based on the model's fields + # This is probably done better in the actual template, we shouldn't + # return Markup straight away. + # Also think of the localization. + return Markup() + + def read_from_request(self, data: Mapping[str, str]): + """Parses the parameters from the given request data. + + :param prefix: The prefix of the input parameter names. + :param data: The request data, e.g. from the POST values. + """ + # TODO: Implement parsing + + +class Transformer(ABC): + """A :class:`Transformer` is the main interface for track manipulation.""" + + @classmethod + @abstractmethod + def identifier(cls) -> str: + """Returns a string identifier for this transformer. + + This identifier is used when serializing/deserializing the filters. + + :return: A machine-readable identifier for this transformer. + """ + + @classmethod + @abstractmethod + def name(cls) -> TranslationString: + """The human-readable name of this transformer, as a translateable string. + + :return: The transformer's name. + """ + + @classmethod + @abstractmethod + def parameter_model(cls) -> type[Parameters]: + """Returns the parameter model that this transformer expects. + + :return: The parameter model class. + """ + + @property + @abstractmethod + def parameters(self) -> Parameters: + """Returns the parameters of this transformer. + + Note that the caller may modify the parameters, which should be + reflected in future applications of the transformer. + + :return: The parameters. + """ + + @parameters.setter + @abstractmethod + def parameters(self, value: Parameters): + pass + + @abstractmethod + def execute(self, gpx: GPX): + """Run the transformation on the input gpx. + + This is expected to modify the GPX object to represent the new state. + + :param gpx: The GPX object to transform. Note that this object will be + mutated! + """ + + +class FixNullElevation(Transformer): + """A transformer that fixes points with zero elevation.""" + + @classmethod + def identifier(cls) -> str: + return "fix-null-elevation" + + @classmethod + def name(cls) -> TranslationString: + return _("transformers.fix-null-elevation") + + @classmethod + def parameter_model(cls) -> type[Parameters]: + return Parameters + + @property + def parameters(self) -> Parameters: + return Parameters() + + @parameters.setter + def parameters(self, value): + pass + + def execute(self, gpx): + print("YALLA YALLA") + + +def list_transformers() -> list[type[Transformer]]: + """Returns a list of all available transformers. + + :return: A list of transformers. + """ + return [ + FixNullElevation, + ] + + +__all__ = ["Parameters", "Transformer", "list_transformers"] diff --git a/fietsboek/views/edit.py b/fietsboek/views/edit.py index e3fe99d..d60cced 100644 --- a/fietsboek/views/edit.py +++ b/fietsboek/views/edit.py @@ -95,6 +95,7 @@ def do_edit(request): track.sync_tags(tags) actions.edit_images(request, request.context) + actions.execute_transformers(request, request.context) data.engrave_metadata( title=track.title, description=track.description, -- cgit v1.2.3