initial work on transformers

So far it doesn't really do much yet, but it does have the machinery to
list the available transformers and run them. It also memorizes if the
transformers even need to be run at all, to save time if the current
configuration already matches.

The parameter UI still needs some work. This is fine because the first
transformer will not have any parameters (it's just the elevation fix).
We probably don't want to have a method that returns Markup, as that
makes it hard to implement localization in there, and the method would
need to be aware of bootstrap.

Another point to think about is documentation. I'd like some information
for the user what the "transformers" are, so we'll probably add a small
tagline and later extend the documentation with some more information (I
want a user chapter in there at some point anyway).

7 files changed, 282 insertions, 4 deletions

diff --git a/fietsboek/__init__.py b/fietsboek/__init__.py
index 9112576..95ff394 100644
--- a/fietsboek/__init__.py
+++ b/fietsboek/__init__.py
@@ -29,6 +29,7 @@ from pyramid.session import SignedCookieSessionFactory
 
 from . import config as mod_config
 from . import jinja2 as mod_jinja2
+from . import transformers
 from .data import DataManager
 from .pages import Pages
 from .security import SecurityPolicy
@@ -138,5 +139,6 @@ def main(_global_config, **settings):
     jinja2_env.filters["format_datetime"] = mod_jinja2.filter_format_datetime
     jinja2_env.filters["local_datetime"] = mod_jinja2.filter_local_datetime
     jinja2_env.globals["embed_tile_layers"] = mod_jinja2.global_embed_tile_layers
+    jinja2_env.globals["list_transformers"] = transformers.list_transformers
 
     return config.make_wsgi_app()
diff --git a/fietsboek/actions.py b/fietsboek/actions.py
index f1a32fc..350faaf 100644
--- a/fietsboek/actions.py
+++ b/fietsboek/actions.py
@@ -10,13 +10,15 @@ import logging
 import re
 from typing import List
 
+import brotli
+import gpxpy
 from pyramid.request import Request
 from sqlalchemy import select
 from sqlalchemy.orm.session import Session
 
-from fietsboek import models, util
-from fietsboek.data import DataManager
-from fietsboek.models.track import TrackType, Visibility
+from . import models, transformers, util
+from .data import DataManager
+from .models.track import TrackType, Visibility
 
 LOGGER = logging.getLogger(__name__)
 
@@ -154,3 +156,58 @@ def edit_images(request: Request, track: models.Track):
         image_meta = models.ImageMetadata.get_or_create(request.dbsession, track, image_id)
         image_meta.description = description
         request.dbsession.add(image_meta)
+
+
+def execute_transformers(request: Request, track: models.Track):
+    """Execute the transformers for the given track.
+
+    Note that this function "short circuits" if the saved transformer settings
+    already match the settings given in the request.
+
+    :param request: The request.
+    :param track: The track.
+    """
+    # pylint: disable=too-many-locals
+    LOGGER.debug("Executing transformers for %d", track.id)
+
+    settings = []
+    for tfm in transformers.list_transformers():
+        ident = tfm.identifier()
+        prefix = f"transformer[{ident}]"
+        req_params = {}
+        for name, val in request.params.items():
+            if name.startswith(prefix):
+                name = name[len(prefix) :]
+                name = name.strip("[]")
+                req_params[name] = val
+
+        if req_params.get("") == "on":
+            params = tfm().parameters
+            params.read_from_request(req_params)
+            settings.append((ident, params))
+
+    serialized = [[tfm_id, params.dict()] for tfm_id, params in settings]
+    if serialized == track.transformers:
+        LOGGER.debug("Applied transformations mach on %d, skipping", track.id)
+        return
+
+    # We always start with the backup, that way we don't get "deepfried GPX"
+    # files by having the same filters run multiple times on the same input.
+    # They are not idempotent after all.
+    manager = request.data_manager.open(track.id)
+    gpx_bytes = manager.backup_path().read_bytes()
+    gpx_bytes = brotli.decompress(gpx_bytes)
+    gpx = gpxpy.parse(gpx_bytes)
+
+    tfms = {tfm.identifier(): tfm for tfm in transformers.list_transformers()}
+    for tfm_id, params in settings:
+        transformer = tfms[tfm_id]()
+        transformer.parameters = params
+        LOGGER.debug("Running %s with %r", transformer, params)
+        transformer.execute(gpx)
+
+    LOGGER.debug("Saving transformed file for %d", track.id)
+    manager.compress_gpx(gpx.to_xml().encode("utf-8"))
+
+    LOGGER.debug("Saving new transformers on %d", track.id)
+    track.transformers = serialized
diff --git a/fietsboek/alembic/versions/20230203_3149aa2d0114.py b/fietsboek/alembic/versions/20230203_3149aa2d0114.py
new file mode 100644
index 0000000..eb9ef78
--- /dev/null
+++ b/fietsboek/alembic/versions/20230203_3149aa2d0114.py
@@ -0,0 +1,22 @@
+"""add transformer column
+
+Revision ID: 3149aa2d0114
+Revises: c939800af428
+Create Date: 2023-02-03 21:44:39.429564
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '3149aa2d0114'
+down_revision = 'c939800af428'
+branch_labels = None
+depends_on = None
+
+def upgrade():
+    op.add_column('tracks', sa.Column('transformers', sa.JSON(), nullable=True))
+    op.execute('UPDATE tracks SET transformers="[]";')
+
+def downgrade():
+    op.drop_column('tracks', 'transformers')
diff --git a/fietsboek/models/track.py b/fietsboek/models/track.py
index 9f9d7f6..a7217bf 100644
--- a/fietsboek/models/track.py
+++ b/fietsboek/models/track.py
@@ -32,6 +32,7 @@ from pyramid.httpexceptions import HTTPNotFound
 from pyramid.i18n import Localizer
 from pyramid.i18n import TranslationString as _
 from sqlalchemy import (
+    JSON,
     Column,
     DateTime,
     Enum,
@@ -174,6 +175,8 @@ class Track(Base):
     :vartype link_secret: str
     :ivar type: Type of the track
     :vartype type: TrackType
+    :ivar transformers: The enabled transformers together with their parameters.
+    :vartype transformers: list[tuple[str, dict]]
     :ivar owner: Owner of the track.
     :vartype owner: fietsboek.models.user.User
     :ivar cache: Cache for the computed track metadata.
@@ -198,6 +201,7 @@ class Track(Base):
     visibility = Column(Enum(Visibility))
     link_secret = Column(Text)
     type = Column(Enum(TrackType))
+    transformers = Column(JSON)
 
     owner = relationship("User", back_populates="tracks")
     cache = relationship(
@@ -397,12 +401,25 @@ class Track(Base):
                 self.tags.append(Tag(tag=to_add))
 
         to_delete = []
-        for (i, tag) in enumerate(self.tags):
+        for i, tag in enumerate(self.tags):
             if tag.tag.lower() not in lower_tags:
                 to_delete.append(i)
         for i in to_delete[::-1]:
             del self.tags[i]
 
+    def transformer_params_for(self, transformer_id: str) -> Optional[dict]:
+        """Returns the transformer parameters for the given transformer.
+
+        If the transformer is not active, returns ``None``.
+
+        :param transformer_id: The string ID of the transformer.
+        :return: The settings as a dictionary.
+        """
+        for t_id, settings in self.transformers:
+            if t_id == transformer_id:
+                return settings
+        return None
+
 
 class TrackWithMetadata:
     """A class to add metadata to a :class:`Track`.
diff --git a/fietsboek/templates/edit_form.jinja2 b/fietsboek/templates/edit_form.jinja2
index bfd45a1..67c01c9 100644
--- a/fietsboek/templates/edit_form.jinja2
+++ b/fietsboek/templates/edit_form.jinja2
@@ -142,4 +142,38 @@
     </div>
   </div>
 </div>
+
+<div class="mb-3">
+  <div class="accordion accordion-flush">
+    {% for transformer in list_transformers() %}
+    <div class="accordion-item">
+      <h2 class="accordion-header" id="transformer-heading-{{ loop.index }}">
+        <button class="accordion-button collapsed" type="button" data-bs-toggle="collapse" data-bs-target="#transformer-{{ loop.index }}" aria-expanded="true" aria-controls="transformer-{{ loop.index }}">
+        {{ _(transformer.name()) }}
+        </button>
+      </h2>
+    </div>
+    <div id="transformer-{{ loop.index}}" class="accordion-collapse collapse" aria-labelledby="transformer-heading-{{ loop.index }}">
+      <div class="accordion-body">
+        {% set params = track.transformer_params_for(transformer.identifier()) %}
+
+        <!-- Checkbox to enable the transformer -->
+        <div class="form-check">
+          <input class="form-check-input" type="checkbox" value="on" id="transformer-enabled-{{ loop.index }}" name="transformer[{{ transformer.identifier() }}]"{% if params is not none %} checked{% endif %}>
+          <label class="form-check-label" for="transformer-enabled-{{ loop.index }}">
+            {{ _("page.track.form.transformer.enable") }}
+          </label>
+        </div>
+
+        <!-- Parameters as defined by the transformer -->
+        {% if params is not none %}
+        {{ transformer.parameter_model().parse_obj(params).html_ui("transformer[{}][%%]".format(transformer.identifier())) }}
+        {% else %}
+        {{ transformer().parameters.html_ui("transformer[{}][%%]".format(transformer.identifier())) }}
+        {% endif %}
+      </div>
+    </div>
+    {% endfor %}
+  </div>
+</div>
 {% endmacro %}
diff --git a/fietsboek/transformers/__init__.py b/fietsboek/transformers/__init__.py
new file mode 100644
index 0000000..b4ed532
--- /dev/null
+++ b/fietsboek/transformers/__init__.py
@@ -0,0 +1,145 @@
+"""Fietsboek GPX transformers.
+
+A "transformer" is something like a filter - it takes in a GPX file and applies
+some small corrections, such as smoothing out the elevation. In order to avoid
+confusion with the naming (and the "filters" you have when searching for
+tracks), we call those "GPX filters" *transformers*.
+
+This module defines the abstract interface for transformers, as well as
+function to load and apply transformers.
+"""
+
+from abc import ABC, abstractmethod
+from collections.abc import Mapping
+
+from gpxpy.gpx import GPX
+from markupsafe import Markup
+from pydantic import BaseModel
+from pyramid.i18n import TranslationString
+
+_ = TranslationString
+
+
+class Parameters(BaseModel):
+    """Parameters for a transformer.
+
+    This is basically a wrapper around pydantic models that allows the
+    parameters to be serialized from and to POST request parameters.
+    """
+
+    def html_ui(self, name_template: str) -> Markup:
+        """Renders a HTML UI for this parameter set.
+
+        :param name_template: The template for the HTML form element names,
+            with a %% placeholder for the parameter name.
+        :return: The rendered UI, ready for inclusion.
+        """
+        # TODO: Implement this based on the model's fields
+        # This is probably done better in the actual template, we shouldn't
+        # return Markup straight away.
+        # Also think of the localization.
+        return Markup()
+
+    def read_from_request(self, data: Mapping[str, str]):
+        """Parses the parameters from the given request data.
+
+        :param prefix: The prefix of the input parameter names.
+        :param data: The request data, e.g. from the POST values.
+        """
+        # TODO: Implement parsing
+
+
+class Transformer(ABC):
+    """A :class:`Transformer` is the main interface for track manipulation."""
+
+    @classmethod
+    @abstractmethod
+    def identifier(cls) -> str:
+        """Returns a string identifier for this transformer.
+
+        This identifier is used when serializing/deserializing the filters.
+
+        :return: A machine-readable identifier for this transformer.
+        """
+
+    @classmethod
+    @abstractmethod
+    def name(cls) -> TranslationString:
+        """The human-readable name of this transformer, as a translateable string.
+
+        :return: The transformer's name.
+        """
+
+    @classmethod
+    @abstractmethod
+    def parameter_model(cls) -> type[Parameters]:
+        """Returns the parameter model that this transformer expects.
+
+        :return: The parameter model class.
+        """
+
+    @property
+    @abstractmethod
+    def parameters(self) -> Parameters:
+        """Returns the parameters of this transformer.
+
+        Note that the caller may modify the parameters, which should be
+        reflected in future applications of the transformer.
+
+        :return: The parameters.
+        """
+
+    @parameters.setter
+    @abstractmethod
+    def parameters(self, value: Parameters):
+        pass
+
+    @abstractmethod
+    def execute(self, gpx: GPX):
+        """Run the transformation on the input gpx.
+
+        This is expected to modify the GPX object to represent the new state.
+
+        :param gpx: The GPX object to transform. Note that this object will be
+            mutated!
+        """
+
+
+class FixNullElevation(Transformer):
+    """A transformer that fixes points with zero elevation."""
+
+    @classmethod
+    def identifier(cls) -> str:
+        return "fix-null-elevation"
+
+    @classmethod
+    def name(cls) -> TranslationString:
+        return _("transformers.fix-null-elevation")
+
+    @classmethod
+    def parameter_model(cls) -> type[Parameters]:
+        return Parameters
+
+    @property
+    def parameters(self) -> Parameters:
+        return Parameters()
+
+    @parameters.setter
+    def parameters(self, value):
+        pass
+
+    def execute(self, gpx):
+        print("YALLA YALLA")
+
+
+def list_transformers() -> list[type[Transformer]]:
+    """Returns a list of all available transformers.
+
+    :return: A list of transformers.
+    """
+    return [
+        FixNullElevation,
+    ]
+
+
+__all__ = ["Parameters", "Transformer", "list_transformers"]
diff --git a/fietsboek/views/edit.py b/fietsboek/views/edit.py
index e3fe99d..d60cced 100644
--- a/fietsboek/views/edit.py
+++ b/fietsboek/views/edit.py
@@ -95,6 +95,7 @@ def do_edit(request):
         track.sync_tags(tags)
 
         actions.edit_images(request, request.context)
+        actions.execute_transformers(request, request.context)
         data.engrave_metadata(
             title=track.title,
             description=track.description,