From aebb3cb0bf39f47bd3c1cf44a2a3605405a2de59 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Fri, 3 Feb 2023 22:44:49 +0100 Subject: initial work on transformers So far it doesn't really do much yet, but it does have the machinery to list the available transformers and run them. It also memorizes if the transformers even need to be run at all, to save time if the current configuration already matches. The parameter UI still needs some work. This is fine because the first transformer will not have any parameters (it's just the elevation fix). We probably don't want to have a method that returns Markup, as that makes it hard to implement localization in there, and the method would need to be aware of bootstrap. Another point to think about is documentation. I'd like some information for the user what the "transformers" are, so we'll probably add a small tagline and later extend the documentation with some more information (I want a user chapter in there at some point anyway). --- fietsboek/__init__.py | 2 + fietsboek/actions.py | 63 ++++++++- .../alembic/versions/20230203_3149aa2d0114.py | 22 ++++ fietsboek/models/track.py | 19 ++- fietsboek/templates/edit_form.jinja2 | 34 +++++ fietsboek/transformers/__init__.py | 145 +++++++++++++++++++++ fietsboek/views/edit.py | 1 + 7 files changed, 282 insertions(+), 4 deletions(-) create mode 100644 fietsboek/alembic/versions/20230203_3149aa2d0114.py create mode 100644 fietsboek/transformers/__init__.py diff --git a/fietsboek/__init__.py b/fietsboek/__init__.py index 9112576..95ff394 100644 --- a/fietsboek/__init__.py +++ b/fietsboek/__init__.py @@ -29,6 +29,7 @@ from pyramid.session import SignedCookieSessionFactory from . import config as mod_config from . import jinja2 as mod_jinja2 +from . import transformers from .data import DataManager from .pages import Pages from .security import SecurityPolicy @@ -138,5 +139,6 @@ def main(_global_config, **settings): jinja2_env.filters["format_datetime"] = mod_jinja2.filter_format_datetime jinja2_env.filters["local_datetime"] = mod_jinja2.filter_local_datetime jinja2_env.globals["embed_tile_layers"] = mod_jinja2.global_embed_tile_layers + jinja2_env.globals["list_transformers"] = transformers.list_transformers return config.make_wsgi_app() diff --git a/fietsboek/actions.py b/fietsboek/actions.py index f1a32fc..350faaf 100644 --- a/fietsboek/actions.py +++ b/fietsboek/actions.py @@ -10,13 +10,15 @@ import logging import re from typing import List +import brotli +import gpxpy from pyramid.request import Request from sqlalchemy import select from sqlalchemy.orm.session import Session -from fietsboek import models, util -from fietsboek.data import DataManager -from fietsboek.models.track import TrackType, Visibility +from . import models, transformers, util +from .data import DataManager +from .models.track import TrackType, Visibility LOGGER = logging.getLogger(__name__) @@ -154,3 +156,58 @@ def edit_images(request: Request, track: models.Track): image_meta = models.ImageMetadata.get_or_create(request.dbsession, track, image_id) image_meta.description = description request.dbsession.add(image_meta) + + +def execute_transformers(request: Request, track: models.Track): + """Execute the transformers for the given track. + + Note that this function "short circuits" if the saved transformer settings + already match the settings given in the request. + + :param request: The request. + :param track: The track. + """ + # pylint: disable=too-many-locals + LOGGER.debug("Executing transformers for %d", track.id) + + settings = [] + for tfm in transformers.list_transformers(): + ident = tfm.identifier() + prefix = f"transformer[{ident}]" + req_params = {} + for name, val in request.params.items(): + if name.startswith(prefix): + name = name[len(prefix) :] + name = name.strip("[]") + req_params[name] = val + + if req_params.get("") == "on": + params = tfm().parameters + params.read_from_request(req_params) + settings.append((ident, params)) + + serialized = [[tfm_id, params.dict()] for tfm_id, params in settings] + if serialized == track.transformers: + LOGGER.debug("Applied transformations mach on %d, skipping", track.id) + return + + # We always start with the backup, that way we don't get "deepfried GPX" + # files by having the same filters run multiple times on the same input. + # They are not idempotent after all. + manager = request.data_manager.open(track.id) + gpx_bytes = manager.backup_path().read_bytes() + gpx_bytes = brotli.decompress(gpx_bytes) + gpx = gpxpy.parse(gpx_bytes) + + tfms = {tfm.identifier(): tfm for tfm in transformers.list_transformers()} + for tfm_id, params in settings: + transformer = tfms[tfm_id]() + transformer.parameters = params + LOGGER.debug("Running %s with %r", transformer, params) + transformer.execute(gpx) + + LOGGER.debug("Saving transformed file for %d", track.id) + manager.compress_gpx(gpx.to_xml().encode("utf-8")) + + LOGGER.debug("Saving new transformers on %d", track.id) + track.transformers = serialized diff --git a/fietsboek/alembic/versions/20230203_3149aa2d0114.py b/fietsboek/alembic/versions/20230203_3149aa2d0114.py new file mode 100644 index 0000000..eb9ef78 --- /dev/null +++ b/fietsboek/alembic/versions/20230203_3149aa2d0114.py @@ -0,0 +1,22 @@ +"""add transformer column + +Revision ID: 3149aa2d0114 +Revises: c939800af428 +Create Date: 2023-02-03 21:44:39.429564 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = '3149aa2d0114' +down_revision = 'c939800af428' +branch_labels = None +depends_on = None + +def upgrade(): + op.add_column('tracks', sa.Column('transformers', sa.JSON(), nullable=True)) + op.execute('UPDATE tracks SET transformers="[]";') + +def downgrade(): + op.drop_column('tracks', 'transformers') diff --git a/fietsboek/models/track.py b/fietsboek/models/track.py index 9f9d7f6..a7217bf 100644 --- a/fietsboek/models/track.py +++ b/fietsboek/models/track.py @@ -32,6 +32,7 @@ from pyramid.httpexceptions import HTTPNotFound from pyramid.i18n import Localizer from pyramid.i18n import TranslationString as _ from sqlalchemy import ( + JSON, Column, DateTime, Enum, @@ -174,6 +175,8 @@ class Track(Base): :vartype link_secret: str :ivar type: Type of the track :vartype type: TrackType + :ivar transformers: The enabled transformers together with their parameters. + :vartype transformers: list[tuple[str, dict]] :ivar owner: Owner of the track. :vartype owner: fietsboek.models.user.User :ivar cache: Cache for the computed track metadata. @@ -198,6 +201,7 @@ class Track(Base): visibility = Column(Enum(Visibility)) link_secret = Column(Text) type = Column(Enum(TrackType)) + transformers = Column(JSON) owner = relationship("User", back_populates="tracks") cache = relationship( @@ -397,12 +401,25 @@ class Track(Base): self.tags.append(Tag(tag=to_add)) to_delete = [] - for (i, tag) in enumerate(self.tags): + for i, tag in enumerate(self.tags): if tag.tag.lower() not in lower_tags: to_delete.append(i) for i in to_delete[::-1]: del self.tags[i] + def transformer_params_for(self, transformer_id: str) -> Optional[dict]: + """Returns the transformer parameters for the given transformer. + + If the transformer is not active, returns ``None``. + + :param transformer_id: The string ID of the transformer. + :return: The settings as a dictionary. + """ + for t_id, settings in self.transformers: + if t_id == transformer_id: + return settings + return None + class TrackWithMetadata: """A class to add metadata to a :class:`Track`. diff --git a/fietsboek/templates/edit_form.jinja2 b/fietsboek/templates/edit_form.jinja2 index bfd45a1..67c01c9 100644 --- a/fietsboek/templates/edit_form.jinja2 +++ b/fietsboek/templates/edit_form.jinja2 @@ -142,4 +142,38 @@ + +
+
+ {% for transformer in list_transformers() %} +
+

+ +

+
+
+
+ {% set params = track.transformer_params_for(transformer.identifier()) %} + + +
+ + +
+ + + {% if params is not none %} + {{ transformer.parameter_model().parse_obj(params).html_ui("transformer[{}][%%]".format(transformer.identifier())) }} + {% else %} + {{ transformer().parameters.html_ui("transformer[{}][%%]".format(transformer.identifier())) }} + {% endif %} +
+
+ {% endfor %} +
+
{% endmacro %} diff --git a/fietsboek/transformers/__init__.py b/fietsboek/transformers/__init__.py new file mode 100644 index 0000000..b4ed532 --- /dev/null +++ b/fietsboek/transformers/__init__.py @@ -0,0 +1,145 @@ +"""Fietsboek GPX transformers. + +A "transformer" is something like a filter - it takes in a GPX file and applies +some small corrections, such as smoothing out the elevation. In order to avoid +confusion with the naming (and the "filters" you have when searching for +tracks), we call those "GPX filters" *transformers*. + +This module defines the abstract interface for transformers, as well as +function to load and apply transformers. +""" + +from abc import ABC, abstractmethod +from collections.abc import Mapping + +from gpxpy.gpx import GPX +from markupsafe import Markup +from pydantic import BaseModel +from pyramid.i18n import TranslationString + +_ = TranslationString + + +class Parameters(BaseModel): + """Parameters for a transformer. + + This is basically a wrapper around pydantic models that allows the + parameters to be serialized from and to POST request parameters. + """ + + def html_ui(self, name_template: str) -> Markup: + """Renders a HTML UI for this parameter set. + + :param name_template: The template for the HTML form element names, + with a %% placeholder for the parameter name. + :return: The rendered UI, ready for inclusion. + """ + # TODO: Implement this based on the model's fields + # This is probably done better in the actual template, we shouldn't + # return Markup straight away. + # Also think of the localization. + return Markup() + + def read_from_request(self, data: Mapping[str, str]): + """Parses the parameters from the given request data. + + :param prefix: The prefix of the input parameter names. + :param data: The request data, e.g. from the POST values. + """ + # TODO: Implement parsing + + +class Transformer(ABC): + """A :class:`Transformer` is the main interface for track manipulation.""" + + @classmethod + @abstractmethod + def identifier(cls) -> str: + """Returns a string identifier for this transformer. + + This identifier is used when serializing/deserializing the filters. + + :return: A machine-readable identifier for this transformer. + """ + + @classmethod + @abstractmethod + def name(cls) -> TranslationString: + """The human-readable name of this transformer, as a translateable string. + + :return: The transformer's name. + """ + + @classmethod + @abstractmethod + def parameter_model(cls) -> type[Parameters]: + """Returns the parameter model that this transformer expects. + + :return: The parameter model class. + """ + + @property + @abstractmethod + def parameters(self) -> Parameters: + """Returns the parameters of this transformer. + + Note that the caller may modify the parameters, which should be + reflected in future applications of the transformer. + + :return: The parameters. + """ + + @parameters.setter + @abstractmethod + def parameters(self, value: Parameters): + pass + + @abstractmethod + def execute(self, gpx: GPX): + """Run the transformation on the input gpx. + + This is expected to modify the GPX object to represent the new state. + + :param gpx: The GPX object to transform. Note that this object will be + mutated! + """ + + +class FixNullElevation(Transformer): + """A transformer that fixes points with zero elevation.""" + + @classmethod + def identifier(cls) -> str: + return "fix-null-elevation" + + @classmethod + def name(cls) -> TranslationString: + return _("transformers.fix-null-elevation") + + @classmethod + def parameter_model(cls) -> type[Parameters]: + return Parameters + + @property + def parameters(self) -> Parameters: + return Parameters() + + @parameters.setter + def parameters(self, value): + pass + + def execute(self, gpx): + print("YALLA YALLA") + + +def list_transformers() -> list[type[Transformer]]: + """Returns a list of all available transformers. + + :return: A list of transformers. + """ + return [ + FixNullElevation, + ] + + +__all__ = ["Parameters", "Transformer", "list_transformers"] diff --git a/fietsboek/views/edit.py b/fietsboek/views/edit.py index e3fe99d..d60cced 100644 --- a/fietsboek/views/edit.py +++ b/fietsboek/views/edit.py @@ -95,6 +95,7 @@ def do_edit(request): track.sync_tags(tags) actions.edit_images(request, request.context) + actions.execute_transformers(request, request.context) data.engrave_metadata( title=track.title, description=track.description, -- cgit v1.2.3 From 94f2f0758a79f797233cfd7e7db241e3eca2e700 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Wed, 8 Feb 2023 23:22:17 +0100 Subject: add a short description to transformers --- fietsboek/templates/edit_form.jinja2 | 2 ++ fietsboek/transformers/__init__.py | 23 +++++++++++++++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/fietsboek/templates/edit_form.jinja2 b/fietsboek/templates/edit_form.jinja2 index 67c01c9..491db98 100644 --- a/fietsboek/templates/edit_form.jinja2 +++ b/fietsboek/templates/edit_form.jinja2 @@ -156,6 +156,8 @@
{% set params = track.transformer_params_for(transformer.identifier()) %} + +

{{ _(transformer.description()) }}

diff --git a/fietsboek/transformers/__init__.py b/fietsboek/transformers/__init__.py index b4ed532..1a9517e 100644 --- a/fietsboek/transformers/__init__.py +++ b/fietsboek/transformers/__init__.py @@ -11,6 +11,7 @@ function to load and apply transformers. from abc import ABC, abstractmethod from collections.abc import Mapping +from typing import TypeVar from gpxpy.gpx import GPX from markupsafe import Markup @@ -19,6 +20,8 @@ from pyramid.i18n import TranslationString _ = TranslationString +T = TypeVar("T", bound="Transformer") + class Parameters(BaseModel): """Parameters for a transformer. @@ -54,7 +57,7 @@ class Transformer(ABC): @classmethod @abstractmethod - def identifier(cls) -> str: + def identifier(cls: type[T]) -> str: """Returns a string identifier for this transformer. This identifier is used when serializing/deserializing the filters. @@ -64,7 +67,7 @@ class Transformer(ABC): @classmethod @abstractmethod - def name(cls) -> TranslationString: + def name(cls: type[T]) -> TranslationString: """The human-readable name of this transformer, as a translateable string. :return: The transformer's name. @@ -72,7 +75,15 @@ class Transformer(ABC): @classmethod @abstractmethod - def parameter_model(cls) -> type[Parameters]: + def description(cls: type[T]) -> TranslationString: + """A short description of what this transformer does. + + :return: The transformer's description. + """ + + @classmethod + @abstractmethod + def parameter_model(cls: type[T]) -> type[Parameters]: """Returns the parameter model that this transformer expects. :return: The parameter model class. @@ -114,7 +125,11 @@ class FixNullElevation(Transformer): @classmethod def name(cls) -> TranslationString: - return _("transformers.fix-null-elevation") + return _("transformers.fix-null-elevation.title") + + @classmethod + def description(cls) -> TranslationString: + return _("transformers.fix-null-elevation.description") @classmethod def parameter_model(cls) -> type[Parameters]: -- cgit v1.2.3 From db470d5f6f9c543a105ef34553beaa74870c3fa8 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Wed, 8 Feb 2023 23:51:46 +0100 Subject: fix transformer handling when uploading a file --- fietsboek/actions.py | 39 +++++++++++++++--------------------- fietsboek/models/track.py | 3 +++ fietsboek/templates/edit_form.jinja2 | 2 +- fietsboek/transformers/__init__.py | 28 ++++++++++++++++++++++++++ fietsboek/views/upload.py | 3 ++- tests/playwright/conftest.py | 1 + tests/playwright/test_basic.py | 4 ++-- 7 files changed, 53 insertions(+), 27 deletions(-) diff --git a/fietsboek/actions.py b/fietsboek/actions.py index 350faaf..7a69913 100644 --- a/fietsboek/actions.py +++ b/fietsboek/actions.py @@ -16,7 +16,9 @@ from pyramid.request import Request from sqlalchemy import select from sqlalchemy.orm.session import Session -from . import models, transformers, util +from . import models +from . import transformers as mod_transformers +from . import util from .data import DataManager from .models.track import TrackType, Visibility @@ -35,6 +37,7 @@ def add_track( badges: List[models.Badge], tagged_people: List[models.User], tags: List[str], + transformers: list[mod_transformers.Transformer], gpx_data: bytes, ) -> models.Track: """Adds a track to the database. @@ -56,6 +59,7 @@ def add_track( :param badges: Badges to attach to the track. :param tagged_people: List of people to tag. :param tags: List of text tags for the track. + :param transformers: List of :class:`~fietsboek.transformers.Transformer` to apply. :param gpx_data: Actual GPX data (uncompressed, straight from the source). :return: The track object that has been inserted into the database. Useful for its ``id`` attribute. @@ -88,6 +92,12 @@ def add_track( manager.compress_gpx(gpx_data) manager.backup() + gpx = gpxpy.parse(gpx_data) + for transformer in transformers: + LOGGER.debug("Running %s with %r", transformer, transformer.parameters) + transformer.execute(gpx) + track.transformers = [[tfm.identifier(), tfm.parameters.dict()] for tfm in transformers] + manager.engrave_metadata( title=track.title, description=track.description, @@ -170,23 +180,9 @@ def execute_transformers(request: Request, track: models.Track): # pylint: disable=too-many-locals LOGGER.debug("Executing transformers for %d", track.id) - settings = [] - for tfm in transformers.list_transformers(): - ident = tfm.identifier() - prefix = f"transformer[{ident}]" - req_params = {} - for name, val in request.params.items(): - if name.startswith(prefix): - name = name[len(prefix) :] - name = name.strip("[]") - req_params[name] = val - - if req_params.get("") == "on": - params = tfm().parameters - params.read_from_request(req_params) - settings.append((ident, params)) - - serialized = [[tfm_id, params.dict()] for tfm_id, params in settings] + settings = mod_transformers.extract_from_request(request) + + serialized = [[tfm.identifier(), tfm.parameters.dict()] for tfm in settings] if serialized == track.transformers: LOGGER.debug("Applied transformations mach on %d, skipping", track.id) return @@ -199,11 +195,8 @@ def execute_transformers(request: Request, track: models.Track): gpx_bytes = brotli.decompress(gpx_bytes) gpx = gpxpy.parse(gpx_bytes) - tfms = {tfm.identifier(): tfm for tfm in transformers.list_transformers()} - for tfm_id, params in settings: - transformer = tfms[tfm_id]() - transformer.parameters = params - LOGGER.debug("Running %s with %r", transformer, params) + for transformer in settings: + LOGGER.debug("Running %s with %r", transformer, transformer.parameters) transformer.execute(gpx) LOGGER.debug("Saving transformed file for %d", track.id) diff --git a/fietsboek/models/track.py b/fietsboek/models/track.py index a7217bf..9342cff 100644 --- a/fietsboek/models/track.py +++ b/fietsboek/models/track.py @@ -415,6 +415,9 @@ class Track(Base): :param transformer_id: The string ID of the transformer. :return: The settings as a dictionary. """ + if not self.transformers: + return None + for t_id, settings in self.transformers: if t_id == transformer_id: return settings diff --git a/fietsboek/templates/edit_form.jinja2 b/fietsboek/templates/edit_form.jinja2 index 491db98..8f6e092 100644 --- a/fietsboek/templates/edit_form.jinja2 +++ b/fietsboek/templates/edit_form.jinja2 @@ -155,7 +155,7 @@
- {% set params = track.transformer_params_for(transformer.identifier()) %} + {% set params = track.transformer_params_for(transformer.identifier()) if track else None %}

{{ _(transformer.description()) }}

diff --git a/fietsboek/transformers/__init__.py b/fietsboek/transformers/__init__.py index 1a9517e..b22c492 100644 --- a/fietsboek/transformers/__init__.py +++ b/fietsboek/transformers/__init__.py @@ -17,6 +17,7 @@ from gpxpy.gpx import GPX from markupsafe import Markup from pydantic import BaseModel from pyramid.i18n import TranslationString +from pyramid.request import Request _ = TranslationString @@ -157,4 +158,31 @@ def list_transformers() -> list[type[Transformer]]: ] +def extract_from_request(request: Request) -> list[Transformer]: + """Extracts the list of transformers to execute from the given request. + + Note that this sets up the transformers with the right parameters. + + :param request: The pyramid request. + :return: The list of prepared transformers. + """ + transformers = [] + for tfm in list_transformers(): + ident = tfm.identifier() + prefix = f"transformer[{ident}]" + req_params = {} + for name, val in request.params.items(): + if name.startswith(prefix): + name = name[len(prefix) :] + name = name.strip("[]") + req_params[name] = val + + if req_params.get("") == "on": + instance = tfm() + instance.parameters.read_from_request(req_params) + transformers.append(instance) + + return transformers + + __all__ = ["Parameters", "Transformer", "list_transformers"] diff --git a/fietsboek/views/upload.py b/fietsboek/views/upload.py index e172339..15fa990 100644 --- a/fietsboek/views/upload.py +++ b/fietsboek/views/upload.py @@ -9,7 +9,7 @@ from pyramid.response import Response from pyramid.view import view_config from sqlalchemy import select -from .. import actions, models, util +from .. import actions, models, transformers, util from ..models.track import TrackType, Visibility LOGGER = logging.getLogger(__name__) @@ -170,6 +170,7 @@ def do_finish_upload(request): tagged_people=tagged_people, date=date, tags=request.params.getall("tag[]"), + transformers=transformers.extract_from_request(request), gpx_data=upload.gpx_data, ) request.dbsession.delete(upload) diff --git a/tests/playwright/conftest.py b/tests/playwright/conftest.py index 18b7ad0..f57aca7 100644 --- a/tests/playwright/conftest.py +++ b/tests/playwright/conftest.py @@ -126,6 +126,7 @@ class Helper: tags=[], badges=[], tagged_people=[], + transformers=[], gpx_data=load_gpx_asset(track_name), ) self.dbaccess.commit() diff --git a/tests/playwright/test_basic.py b/tests/playwright/test_basic.py index 3b3329a..a98e52d 100644 --- a/tests/playwright/test_basic.py +++ b/tests/playwright/test_basic.py @@ -51,7 +51,7 @@ def test_upload(page: Page, playwright_helper, tmp_path, dbaccess): page.locator(".bi-upload").click() # We now fill in most of the data - page.get_by_label("Title").fill("An awesome track!") + page.get_by_label("Title", exact=True).fill("An awesome track!") page.get_by_label("Date").type("07302022") page.get_by_label("Date").press("Tab") page.get_by_label("Date").type("12:41") @@ -82,7 +82,7 @@ def test_edit(page: Page, playwright_helper, dbaccess): page.locator(".btn", has_text="Edit").click() # We now fill in most of the data - page.get_by_label("Title").fill("Not so awesome anymore!") + page.get_by_label("Title", exact=True).fill("Not so awesome anymore!") page.get_by_label("Date").type("09232019") page.get_by_label("Date").press("Tab") page.get_by_label("Date").type("15:28") -- cgit v1.2.3 From 83dcaf4c1eb7e735f9b07e9766c4f4aac61b138a Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Wed, 15 Feb 2023 22:36:11 +0100 Subject: actually save transformed GPX after upload --- fietsboek/actions.py | 1 + 1 file changed, 1 insertion(+) diff --git a/fietsboek/actions.py b/fietsboek/actions.py index 7a69913..a5ba8a0 100644 --- a/fietsboek/actions.py +++ b/fietsboek/actions.py @@ -96,6 +96,7 @@ def add_track( for transformer in transformers: LOGGER.debug("Running %s with %r", transformer, transformer.parameters) transformer.execute(gpx) + manager.compress_gpx(gpx.to_xml().encode("utf-8")) track.transformers = [[tfm.identifier(), tfm.parameters.dict()] for tfm in transformers] manager.engrave_metadata( -- cgit v1.2.3 From c79802acda75e5e7e26c45a3a4d9da6b51da9215 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Wed, 15 Feb 2023 23:08:16 +0100 Subject: implement null-elevation-fix transformer --- fietsboek/locale/de/LC_MESSAGES/messages.mo | Bin 11757 -> 12059 bytes fietsboek/locale/de/LC_MESSAGES/messages.po | 80 +++++++++++++++----------- fietsboek/locale/en/LC_MESSAGES/messages.mo | Bin 11069 -> 11347 bytes fietsboek/locale/en/LC_MESSAGES/messages.po | 80 +++++++++++++++----------- fietsboek/locale/fietslog.pot | 84 ++++++++++++++++------------ fietsboek/transformers/__init__.py | 21 ++++++- 6 files changed, 160 insertions(+), 105 deletions(-) diff --git a/fietsboek/locale/de/LC_MESSAGES/messages.mo b/fietsboek/locale/de/LC_MESSAGES/messages.mo index 7092994..6043d8e 100644 Binary files a/fietsboek/locale/de/LC_MESSAGES/messages.mo and b/fietsboek/locale/de/LC_MESSAGES/messages.mo differ diff --git a/fietsboek/locale/de/LC_MESSAGES/messages.po b/fietsboek/locale/de/LC_MESSAGES/messages.po index dfa2bed..8787ec2 100644 --- a/fietsboek/locale/de/LC_MESSAGES/messages.po +++ b/fietsboek/locale/de/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2022-12-10 17:37+0100\n" +"POT-Creation-Date: 2023-02-15 23:01+0100\n" "PO-Revision-Date: 2022-07-02 17:35+0200\n" "Last-Translator: FULL NAME \n" "Language: de\n" @@ -18,39 +18,39 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 2.11.0\n" -#: fietsboek/util.py:274 +#: fietsboek/util.py:273 msgid "password_constraint.mismatch" msgstr "Passwörter stimmen nicht überein" -#: fietsboek/util.py:276 +#: fietsboek/util.py:275 msgid "password_constraint.length" msgstr "Passwort zu kurz" -#: fietsboek/models/track.py:543 +#: fietsboek/models/track.py:565 msgid "tooltip.table.length" msgstr "Länge" -#: fietsboek/models/track.py:544 +#: fietsboek/models/track.py:566 msgid "tooltip.table.uphill" msgstr "Bergauf" -#: fietsboek/models/track.py:545 +#: fietsboek/models/track.py:567 msgid "tooltip.table.downhill" msgstr "Bergab" -#: fietsboek/models/track.py:546 +#: fietsboek/models/track.py:568 msgid "tooltip.table.moving_time" msgstr "Fahrzeit" -#: fietsboek/models/track.py:547 +#: fietsboek/models/track.py:569 msgid "tooltip.table.stopped_time" msgstr "Haltezeit" -#: fietsboek/models/track.py:549 +#: fietsboek/models/track.py:571 msgid "tooltip.table.max_speed" msgstr "Maximalgeschwindigkeit" -#: fietsboek/models/track.py:553 +#: fietsboek/models/track.py:575 msgid "tooltip.table.avg_speed" msgstr "Durchschnittsgeschwindigkeit" @@ -438,6 +438,10 @@ msgstr "Bildbeschreibung" msgid "page.track.form.image_description_modal.save" msgstr "Übernehmen" +#: fietsboek/templates/edit_form.jinja2:166 +msgid "page.track.form.transformer.enable" +msgstr "Transformation anwenden" + #: fietsboek/templates/finish_upload.jinja2:8 #: fietsboek/templates/upload.jinja2:6 msgid "page.upload.title" @@ -460,47 +464,47 @@ msgstr "Startseite" msgid "page.home.total" msgstr "Gesamt" -#: fietsboek/templates/layout.jinja2:35 +#: fietsboek/templates/layout.jinja2:36 msgid "page.navbar.toggle" msgstr "Navigation umschalten" -#: fietsboek/templates/layout.jinja2:46 +#: fietsboek/templates/layout.jinja2:47 msgid "page.navbar.home" msgstr "Startseite" -#: fietsboek/templates/layout.jinja2:49 +#: fietsboek/templates/layout.jinja2:50 msgid "page.navbar.browse" msgstr "Stöbern" -#: fietsboek/templates/layout.jinja2:53 +#: fietsboek/templates/layout.jinja2:54 msgid "page.navbar.upload" msgstr "Hochladen" -#: fietsboek/templates/layout.jinja2:62 +#: fietsboek/templates/layout.jinja2:63 msgid "page.navbar.user" msgstr "Nutzer" -#: fietsboek/templates/layout.jinja2:66 +#: fietsboek/templates/layout.jinja2:67 msgid "page.navbar.welcome_user" msgstr "Willkommen, {}!" -#: fietsboek/templates/layout.jinja2:69 +#: fietsboek/templates/layout.jinja2:70 msgid "page.navbar.logout" msgstr "Abmelden" -#: fietsboek/templates/layout.jinja2:72 +#: fietsboek/templates/layout.jinja2:73 msgid "page.navbar.profile" msgstr "Profil" -#: fietsboek/templates/layout.jinja2:76 +#: fietsboek/templates/layout.jinja2:77 msgid "page.navbar.admin" msgstr "Admin" -#: fietsboek/templates/layout.jinja2:82 +#: fietsboek/templates/layout.jinja2:83 msgid "page.navbar.login" msgstr "Anmelden" -#: fietsboek/templates/layout.jinja2:86 +#: fietsboek/templates/layout.jinja2:87 msgid "page.navbar.create_account" msgstr "Konto Erstellen" @@ -626,6 +630,14 @@ msgstr "Anfrage senden" msgid "page.upload.form.gpx" msgstr "GPX Datei" +#: fietsboek/transformers/__init__.py:130 +msgid "transformers.fix-null-elevation.title" +msgstr "Nullhöhen beheben" + +#: fietsboek/transformers/__init__.py:134 +msgid "transformers.fix-null-elevation.description" +msgstr "Diese Transformation passt die Höhenangabe für Punkte an, bei denen die Höhenangabe fehlt." + #: fietsboek/views/account.py:54 msgid "flash.invalid_name" msgstr "Ungültiger Name" @@ -649,15 +661,15 @@ msgstr "" msgid "flash.a_confirmation_link_has_been_sent" msgstr "Ein Bestätigungslink wurde versandt" -#: fietsboek/views/admin.py:49 +#: fietsboek/views/admin.py:48 msgid "flash.badge_added" msgstr "Wappen hinzugefügt" -#: fietsboek/views/admin.py:73 +#: fietsboek/views/admin.py:72 msgid "flash.badge_modified" msgstr "Wappen bearbeitet" -#: fietsboek/views/admin.py:93 +#: fietsboek/views/admin.py:92 msgid "flash.badge_deleted" msgstr "Wappen gelöscht" @@ -705,43 +717,43 @@ msgstr "E-Mail-Adresse bestätigt" msgid "flash.password_updated" msgstr "Passwort aktualisiert" -#: fietsboek/views/detail.py:101 +#: fietsboek/views/detail.py:140 msgid "flash.track_deleted" msgstr "Strecke gelöscht" -#: fietsboek/views/profile.py:61 +#: fietsboek/views/profile.py:60 msgid "flash.personal_data_updated" msgstr "Persönliche Daten wurden gespeichert" -#: fietsboek/views/profile.py:79 +#: fietsboek/views/profile.py:78 msgid "flash.friend_not_found" msgstr "Das angegebene Konto wurde nicht gefunden" -#: fietsboek/views/profile.py:85 +#: fietsboek/views/profile.py:84 msgid "flash.friend_already_exists" msgstr "Dieser Freund existiert bereits" -#: fietsboek/views/profile.py:93 +#: fietsboek/views/profile.py:92 msgid "flash.friend_added" msgstr "Freund hinzugefügt" -#: fietsboek/views/profile.py:103 +#: fietsboek/views/profile.py:102 msgid "flash.friend_request_sent" msgstr "Freundschaftsanfrage gesendet" -#: fietsboek/views/upload.py:56 +#: fietsboek/views/upload.py:52 msgid "flash.no_file_selected" msgstr "Keine Datei ausgewählt" -#: fietsboek/views/upload.py:66 +#: fietsboek/views/upload.py:62 msgid "flash.invalid_file" msgstr "Ungültige GPX-Datei gesendet" -#: fietsboek/views/upload.py:189 +#: fietsboek/views/upload.py:182 msgid "flash.upload_success" msgstr "Hochladen erfolgreich" -#: fietsboek/views/upload.py:205 +#: fietsboek/views/upload.py:198 msgid "flash.upload_cancelled" msgstr "Hochladen abgebrochen" diff --git a/fietsboek/locale/en/LC_MESSAGES/messages.mo b/fietsboek/locale/en/LC_MESSAGES/messages.mo index 7827a62..4f35491 100644 Binary files a/fietsboek/locale/en/LC_MESSAGES/messages.mo and b/fietsboek/locale/en/LC_MESSAGES/messages.mo differ diff --git a/fietsboek/locale/en/LC_MESSAGES/messages.po b/fietsboek/locale/en/LC_MESSAGES/messages.po index f4b0239..5f3ab68 100644 --- a/fietsboek/locale/en/LC_MESSAGES/messages.po +++ b/fietsboek/locale/en/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2022-12-10 17:37+0100\n" +"POT-Creation-Date: 2023-02-15 23:01+0100\n" "PO-Revision-Date: 2022-06-28 13:11+0200\n" "Last-Translator: \n" "Language: en\n" @@ -18,39 +18,39 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 2.11.0\n" -#: fietsboek/util.py:274 +#: fietsboek/util.py:273 msgid "password_constraint.mismatch" msgstr "Passwords don't match" -#: fietsboek/util.py:276 +#: fietsboek/util.py:275 msgid "password_constraint.length" msgstr "Password not long enough" -#: fietsboek/models/track.py:543 +#: fietsboek/models/track.py:565 msgid "tooltip.table.length" msgstr "Length" -#: fietsboek/models/track.py:544 +#: fietsboek/models/track.py:566 msgid "tooltip.table.uphill" msgstr "Uphill" -#: fietsboek/models/track.py:545 +#: fietsboek/models/track.py:567 msgid "tooltip.table.downhill" msgstr "Downhill" -#: fietsboek/models/track.py:546 +#: fietsboek/models/track.py:568 msgid "tooltip.table.moving_time" msgstr "Moving Time" -#: fietsboek/models/track.py:547 +#: fietsboek/models/track.py:569 msgid "tooltip.table.stopped_time" msgstr "Stopped Time" -#: fietsboek/models/track.py:549 +#: fietsboek/models/track.py:571 msgid "tooltip.table.max_speed" msgstr "Max Speed" -#: fietsboek/models/track.py:553 +#: fietsboek/models/track.py:575 msgid "tooltip.table.avg_speed" msgstr "Average Speed" @@ -434,6 +434,10 @@ msgstr "Image description" msgid "page.track.form.image_description_modal.save" msgstr "Apply" +#: fietsboek/templates/edit_form.jinja2:166 +msgid "page.track.form.transformer.enable" +msgstr "Apply transformation" + #: fietsboek/templates/finish_upload.jinja2:8 #: fietsboek/templates/upload.jinja2:6 msgid "page.upload.title" @@ -456,47 +460,47 @@ msgstr "Home" msgid "page.home.total" msgstr "Total" -#: fietsboek/templates/layout.jinja2:35 +#: fietsboek/templates/layout.jinja2:36 msgid "page.navbar.toggle" msgstr "Toggle navigation" -#: fietsboek/templates/layout.jinja2:46 +#: fietsboek/templates/layout.jinja2:47 msgid "page.navbar.home" msgstr "Home" -#: fietsboek/templates/layout.jinja2:49 +#: fietsboek/templates/layout.jinja2:50 msgid "page.navbar.browse" msgstr "Browse" -#: fietsboek/templates/layout.jinja2:53 +#: fietsboek/templates/layout.jinja2:54 msgid "page.navbar.upload" msgstr "Upload" -#: fietsboek/templates/layout.jinja2:62 +#: fietsboek/templates/layout.jinja2:63 msgid "page.navbar.user" msgstr "User" -#: fietsboek/templates/layout.jinja2:66 +#: fietsboek/templates/layout.jinja2:67 msgid "page.navbar.welcome_user" msgstr "Welcome, {}!" -#: fietsboek/templates/layout.jinja2:69 +#: fietsboek/templates/layout.jinja2:70 msgid "page.navbar.logout" msgstr "Logout" -#: fietsboek/templates/layout.jinja2:72 +#: fietsboek/templates/layout.jinja2:73 msgid "page.navbar.profile" msgstr "Profile" -#: fietsboek/templates/layout.jinja2:76 +#: fietsboek/templates/layout.jinja2:77 msgid "page.navbar.admin" msgstr "Admin" -#: fietsboek/templates/layout.jinja2:82 +#: fietsboek/templates/layout.jinja2:83 msgid "page.navbar.login" msgstr "Login" -#: fietsboek/templates/layout.jinja2:86 +#: fietsboek/templates/layout.jinja2:87 msgid "page.navbar.create_account" msgstr "Create Account" @@ -622,6 +626,14 @@ msgstr "Send request" msgid "page.upload.form.gpx" msgstr "GPX file" +#: fietsboek/transformers/__init__.py:130 +msgid "transformers.fix-null-elevation.title" +msgstr "Fix null elevation" + +#: fietsboek/transformers/__init__.py:134 +msgid "transformers.fix-null-elevation.description" +msgstr "This transformer fixes the elevation of points whose elevation is unset." + #: fietsboek/views/account.py:54 msgid "flash.invalid_name" msgstr "Invalid name" @@ -645,15 +657,15 @@ msgstr "" msgid "flash.a_confirmation_link_has_been_sent" msgstr "A confirmation link has been sent" -#: fietsboek/views/admin.py:49 +#: fietsboek/views/admin.py:48 msgid "flash.badge_added" msgstr "Badge has been added" -#: fietsboek/views/admin.py:73 +#: fietsboek/views/admin.py:72 msgid "flash.badge_modified" msgstr "Badge has been modified" -#: fietsboek/views/admin.py:93 +#: fietsboek/views/admin.py:92 msgid "flash.badge_deleted" msgstr "Badge has been deleted" @@ -700,43 +712,43 @@ msgstr "Your email address has been verified" msgid "flash.password_updated" msgstr "Password has been updated" -#: fietsboek/views/detail.py:101 +#: fietsboek/views/detail.py:140 msgid "flash.track_deleted" msgstr "Track has been deleted" -#: fietsboek/views/profile.py:61 +#: fietsboek/views/profile.py:60 msgid "flash.personal_data_updated" msgstr "Personal data has been updated" -#: fietsboek/views/profile.py:79 +#: fietsboek/views/profile.py:78 msgid "flash.friend_not_found" msgstr "The friend was not found" -#: fietsboek/views/profile.py:85 +#: fietsboek/views/profile.py:84 msgid "flash.friend_already_exists" msgstr "Friend already exists" -#: fietsboek/views/profile.py:93 +#: fietsboek/views/profile.py:92 msgid "flash.friend_added" msgstr "Friend has been added" -#: fietsboek/views/profile.py:103 +#: fietsboek/views/profile.py:102 msgid "flash.friend_request_sent" msgstr "Friend request sent" -#: fietsboek/views/upload.py:56 +#: fietsboek/views/upload.py:52 msgid "flash.no_file_selected" msgstr "No file selected" -#: fietsboek/views/upload.py:66 +#: fietsboek/views/upload.py:62 msgid "flash.invalid_file" msgstr "Invalid GPX file selected" -#: fietsboek/views/upload.py:189 +#: fietsboek/views/upload.py:182 msgid "flash.upload_success" msgstr "Upload successful" -#: fietsboek/views/upload.py:205 +#: fietsboek/views/upload.py:198 msgid "flash.upload_cancelled" msgstr "Upload cancelled" diff --git a/fietsboek/locale/fietslog.pot b/fietsboek/locale/fietslog.pot index a86c43b..05bc70a 100644 --- a/fietsboek/locale/fietslog.pot +++ b/fietsboek/locale/fietslog.pot @@ -1,14 +1,14 @@ # Translations template for PROJECT. -# Copyright (C) 2022 ORGANIZATION +# Copyright (C) 2023 ORGANIZATION # This file is distributed under the same license as the PROJECT project. -# FIRST AUTHOR , 2022. +# FIRST AUTHOR , 2023. # #, fuzzy msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2022-12-10 17:37+0100\n" +"POT-Creation-Date: 2023-02-15 23:01+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -17,39 +17,39 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 2.11.0\n" -#: fietsboek/util.py:274 +#: fietsboek/util.py:273 msgid "password_constraint.mismatch" msgstr "" -#: fietsboek/util.py:276 +#: fietsboek/util.py:275 msgid "password_constraint.length" msgstr "" -#: fietsboek/models/track.py:543 +#: fietsboek/models/track.py:565 msgid "tooltip.table.length" msgstr "" -#: fietsboek/models/track.py:544 +#: fietsboek/models/track.py:566 msgid "tooltip.table.uphill" msgstr "" -#: fietsboek/models/track.py:545 +#: fietsboek/models/track.py:567 msgid "tooltip.table.downhill" msgstr "" -#: fietsboek/models/track.py:546 +#: fietsboek/models/track.py:568 msgid "tooltip.table.moving_time" msgstr "" -#: fietsboek/models/track.py:547 +#: fietsboek/models/track.py:569 msgid "tooltip.table.stopped_time" msgstr "" -#: fietsboek/models/track.py:549 +#: fietsboek/models/track.py:571 msgid "tooltip.table.max_speed" msgstr "" -#: fietsboek/models/track.py:553 +#: fietsboek/models/track.py:575 msgid "tooltip.table.avg_speed" msgstr "" @@ -431,6 +431,10 @@ msgstr "" msgid "page.track.form.image_description_modal.save" msgstr "" +#: fietsboek/templates/edit_form.jinja2:166 +msgid "page.track.form.transformer.enable" +msgstr "" + #: fietsboek/templates/finish_upload.jinja2:8 #: fietsboek/templates/upload.jinja2:6 msgid "page.upload.title" @@ -453,47 +457,47 @@ msgstr "" msgid "page.home.total" msgstr "" -#: fietsboek/templates/layout.jinja2:35 +#: fietsboek/templates/layout.jinja2:36 msgid "page.navbar.toggle" msgstr "" -#: fietsboek/templates/layout.jinja2:46 +#: fietsboek/templates/layout.jinja2:47 msgid "page.navbar.home" msgstr "" -#: fietsboek/templates/layout.jinja2:49 +#: fietsboek/templates/layout.jinja2:50 msgid "page.navbar.browse" msgstr "" -#: fietsboek/templates/layout.jinja2:53 +#: fietsboek/templates/layout.jinja2:54 msgid "page.navbar.upload" msgstr "" -#: fietsboek/templates/layout.jinja2:62 +#: fietsboek/templates/layout.jinja2:63 msgid "page.navbar.user" msgstr "" -#: fietsboek/templates/layout.jinja2:66 +#: fietsboek/templates/layout.jinja2:67 msgid "page.navbar.welcome_user" msgstr "" -#: fietsboek/templates/layout.jinja2:69 +#: fietsboek/templates/layout.jinja2:70 msgid "page.navbar.logout" msgstr "" -#: fietsboek/templates/layout.jinja2:72 +#: fietsboek/templates/layout.jinja2:73 msgid "page.navbar.profile" msgstr "" -#: fietsboek/templates/layout.jinja2:76 +#: fietsboek/templates/layout.jinja2:77 msgid "page.navbar.admin" msgstr "" -#: fietsboek/templates/layout.jinja2:82 +#: fietsboek/templates/layout.jinja2:83 msgid "page.navbar.login" msgstr "" -#: fietsboek/templates/layout.jinja2:86 +#: fietsboek/templates/layout.jinja2:87 msgid "page.navbar.create_account" msgstr "" @@ -617,6 +621,14 @@ msgstr "" msgid "page.upload.form.gpx" msgstr "" +#: fietsboek/transformers/__init__.py:130 +msgid "transformers.fix-null-elevation.title" +msgstr "" + +#: fietsboek/transformers/__init__.py:134 +msgid "transformers.fix-null-elevation.description" +msgstr "" + #: fietsboek/views/account.py:54 msgid "flash.invalid_name" msgstr "" @@ -637,15 +649,15 @@ msgstr "" msgid "flash.a_confirmation_link_has_been_sent" msgstr "" -#: fietsboek/views/admin.py:49 +#: fietsboek/views/admin.py:48 msgid "flash.badge_added" msgstr "" -#: fietsboek/views/admin.py:73 +#: fietsboek/views/admin.py:72 msgid "flash.badge_modified" msgstr "" -#: fietsboek/views/admin.py:93 +#: fietsboek/views/admin.py:92 msgid "flash.badge_deleted" msgstr "" @@ -689,43 +701,43 @@ msgstr "" msgid "flash.password_updated" msgstr "" -#: fietsboek/views/detail.py:101 +#: fietsboek/views/detail.py:140 msgid "flash.track_deleted" msgstr "" -#: fietsboek/views/profile.py:61 +#: fietsboek/views/profile.py:60 msgid "flash.personal_data_updated" msgstr "" -#: fietsboek/views/profile.py:79 +#: fietsboek/views/profile.py:78 msgid "flash.friend_not_found" msgstr "" -#: fietsboek/views/profile.py:85 +#: fietsboek/views/profile.py:84 msgid "flash.friend_already_exists" msgstr "" -#: fietsboek/views/profile.py:93 +#: fietsboek/views/profile.py:92 msgid "flash.friend_added" msgstr "" -#: fietsboek/views/profile.py:103 +#: fietsboek/views/profile.py:102 msgid "flash.friend_request_sent" msgstr "" -#: fietsboek/views/upload.py:56 +#: fietsboek/views/upload.py:52 msgid "flash.no_file_selected" msgstr "" -#: fietsboek/views/upload.py:66 +#: fietsboek/views/upload.py:62 msgid "flash.invalid_file" msgstr "" -#: fietsboek/views/upload.py:189 +#: fietsboek/views/upload.py:182 msgid "flash.upload_success" msgstr "" -#: fietsboek/views/upload.py:205 +#: fietsboek/views/upload.py:198 msgid "flash.upload_cancelled" msgstr "" diff --git a/fietsboek/transformers/__init__.py b/fietsboek/transformers/__init__.py index b22c492..1f084a0 100644 --- a/fietsboek/transformers/__init__.py +++ b/fietsboek/transformers/__init__.py @@ -11,6 +11,7 @@ function to load and apply transformers. from abc import ABC, abstractmethod from collections.abc import Mapping +from itertools import chain, islice from typing import TypeVar from gpxpy.gpx import GPX @@ -145,7 +146,25 @@ class FixNullElevation(Transformer): pass def execute(self, gpx): - print("YALLA YALLA") + def all_points(): + return ( + point + for track in gpx.tracks + for segment in track.segments + for point in segment.points + ) + + points = all_points() + previous_points = chain([None], all_points()) + next_points = chain(islice(all_points(), 1, None), [None]) + + for previous_point, point, next_point in zip(previous_points, points, next_points): + if point.elevation == 0.0: + if previous_point: + point.elevation += previous_point.elevation + if next_point: + point.elevation += next_point.elevation + point.elevation /= sum(1 for pt in [previous_point, next_point] if pt) def list_transformers() -> list[type[Transformer]]: -- cgit v1.2.3 From 413ac5ca31dcc3ccb84484748337b815599fe314 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Wed, 15 Feb 2023 23:27:30 +0100 Subject: properly rebuild cache after changing transformers --- fietsboek/actions.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/fietsboek/actions.py b/fietsboek/actions.py index a5ba8a0..4974a2c 100644 --- a/fietsboek/actions.py +++ b/fietsboek/actions.py @@ -81,10 +81,6 @@ def add_track( dbsession.add(track) dbsession.flush() - # Best time to build the cache is right after the upload - track.ensure_cache(gpx_data) - dbsession.add(track.cache) - # Save the GPX data LOGGER.debug("Creating a new data folder for %d", track.id) manager = data_manager.initialize(track.id) @@ -99,6 +95,11 @@ def add_track( manager.compress_gpx(gpx.to_xml().encode("utf-8")) track.transformers = [[tfm.identifier(), tfm.parameters.dict()] for tfm in transformers] + # Best time to build the cache is right after the upload, but *after* the + # transformers have been applied! + track.ensure_cache(manager.decompress_gpx()) + dbsession.add(track.cache) + manager.engrave_metadata( title=track.title, description=track.description, @@ -205,3 +206,9 @@ def execute_transformers(request: Request, track: models.Track): LOGGER.debug("Saving new transformers on %d", track.id) track.transformers = serialized + + LOGGER.debug("Rebuilding cache for %d", track.id) + request.dbsession.delete(track.cache) + track.cache = None + track.ensure_cache(manager.decompress_gpx()) + request.dbsession.add(track.cache) -- cgit v1.2.3 From 4a3ebb1a0a71c02c1057b6fd6c6054afe3bfa876 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Wed, 15 Feb 2023 23:36:55 +0100 Subject: try to avoid parsing the GPX more than once --- fietsboek/actions.py | 16 +++++++++++----- fietsboek/data.py | 12 ++++++++++-- fietsboek/models/track.py | 3 ++- fietsboek/util.py | 9 ++++++--- fietsboek/views/edit.py | 3 ++- 5 files changed, 31 insertions(+), 12 deletions(-) diff --git a/fietsboek/actions.py b/fietsboek/actions.py index 4974a2c..2058f4d 100644 --- a/fietsboek/actions.py +++ b/fietsboek/actions.py @@ -8,7 +8,7 @@ the test functions. import datetime import logging import re -from typing import List +from typing import List, Optional import brotli import gpxpy @@ -97,7 +97,7 @@ def add_track( # Best time to build the cache is right after the upload, but *after* the # transformers have been applied! - track.ensure_cache(manager.decompress_gpx()) + track.ensure_cache(gpx) dbsession.add(track.cache) manager.engrave_metadata( @@ -105,6 +105,7 @@ def add_track( description=track.description, author_name=track.owner.name, time=track.date, + gpx=gpx, ) return track @@ -170,14 +171,18 @@ def edit_images(request: Request, track: models.Track): request.dbsession.add(image_meta) -def execute_transformers(request: Request, track: models.Track): +def execute_transformers(request: Request, track: models.Track) -> Optional[gpxpy.gpx.GPX]: """Execute the transformers for the given track. Note that this function "short circuits" if the saved transformer settings already match the settings given in the request. + This function saves the modified data, but does also return it in case you + need to do further processing (unless no transformations have taken place). + :param request: The request. :param track: The track. + :return: The transformed track. """ # pylint: disable=too-many-locals LOGGER.debug("Executing transformers for %d", track.id) @@ -187,7 +192,7 @@ def execute_transformers(request: Request, track: models.Track): serialized = [[tfm.identifier(), tfm.parameters.dict()] for tfm in settings] if serialized == track.transformers: LOGGER.debug("Applied transformations mach on %d, skipping", track.id) - return + return None # We always start with the backup, that way we don't get "deepfried GPX" # files by having the same filters run multiple times on the same input. @@ -210,5 +215,6 @@ def execute_transformers(request: Request, track: models.Track): LOGGER.debug("Rebuilding cache for %d", track.id) request.dbsession.delete(track.cache) track.cache = None - track.ensure_cache(manager.decompress_gpx()) + track.ensure_cache(gpx) request.dbsession.add(track.cache) + return gpx diff --git a/fietsboek/data.py b/fietsboek/data.py index 1a1b66b..6906c84 100644 --- a/fietsboek/data.py +++ b/fietsboek/data.py @@ -158,7 +158,13 @@ class TrackDataDir: return brotli.decompress(self.gpx_path().read_bytes()) def engrave_metadata( - self, title: str, description: str, author_name: str, time: datetime.datetime + self, + title: str, + description: str, + author_name: str, + time: datetime.datetime, + *, + gpx: Optional[gpxpy.gpx.GPX] = None, ): """Engrave the given metadata into the GPX file. @@ -168,8 +174,10 @@ class TrackDataDir: :param description: The description of the track. :param creator: Name of the track's creator. :param time: Time of the track. + :param gpx: The pre-parsed GPX track, to save time if it is already parsed. """ - gpx = gpxpy.parse(self.decompress_gpx()) + if gpx is None: + gpx = gpxpy.parse(self.decompress_gpx()) # First we delete the existing metadata for track in gpx.tracks: track.name = None diff --git a/fietsboek/models/track.py b/fietsboek/models/track.py index 9342cff..e0d2820 100644 --- a/fietsboek/models/track.py +++ b/fietsboek/models/track.py @@ -18,6 +18,7 @@ import logging from itertools import chain from typing import TYPE_CHECKING, List, Optional, Set, Union +import gpxpy from babel.numbers import format_decimal from markupsafe import Markup from pyramid.authorization import ( @@ -330,7 +331,7 @@ class Track(Base): result = ACLHelper().permits(self, principals, "track.view") return isinstance(result, ACLAllowed) - def ensure_cache(self, gpx_data: Union[str, bytes]): + def ensure_cache(self, gpx_data: Union[str, bytes, gpxpy.gpx.GPX]): """Ensure that a cached version of this track's metadata exists. :param gpx_data: GPX data (uncompressed) from which to build the cache. diff --git a/fietsboek/util.py b/fietsboek/util.py index c741550..68ba769 100644 --- a/fietsboek/util.py +++ b/fietsboek/util.py @@ -152,19 +152,22 @@ def guess_gpx_timezone(gpx: gpxpy.gpx.GPX) -> datetime.tzinfo: return datetime.timezone.utc -def tour_metadata(gpx_data: Union[str, bytes]) -> dict: +def tour_metadata(gpx_data: Union[str, bytes, gpxpy.gpx.GPX]) -> dict: """Calculate the metadata of the tour. Returns a dict with ``length``, ``uphill``, ``downhill``, ``moving_time``, ``stopped_time``, ``max_speed``, ``avg_speed``, ``start_time`` and ``end_time``. - :param gpx_data: The GPX data of the tour. + :param gpx_data: The GPX data of the tour. Can be pre-parsed to save time. :return: A dictionary with the computed values. """ if isinstance(gpx_data, bytes): gpx_data = gpx_data.decode("utf-8") - gpx = gpxpy.parse(gpx_data) + if isinstance(gpx_data, gpxpy.gpx.GPX): + gpx = gpx_data + else: + gpx = gpxpy.parse(gpx_data) timezone = guess_gpx_timezone(gpx) uphill, downhill = gpx.get_uphill_downhill() moving_data = gpx.get_moving_data() diff --git a/fietsboek/views/edit.py b/fietsboek/views/edit.py index d60cced..9cc666f 100644 --- a/fietsboek/views/edit.py +++ b/fietsboek/views/edit.py @@ -95,12 +95,13 @@ def do_edit(request): track.sync_tags(tags) actions.edit_images(request, request.context) - actions.execute_transformers(request, request.context) + gpx = actions.execute_transformers(request, request.context) data.engrave_metadata( title=track.title, description=track.description, author_name=track.owner.name, time=track.date, + gpx=gpx, ) return HTTPFound(request.route_url("details", track_id=track.id)) -- cgit v1.2.3 From b423cc4a2deaa54cf529a1823f5977b5b5ef9b89 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Tue, 21 Feb 2023 00:00:34 +0100 Subject: rework transformer parameter UI logic I don't like interweaving the HTML code into the business logic, so now we can have that in the Jinja template. Ideally, the list of definitions would be generated automatically from the model attributes. --- fietsboek/templates/edit_form.jinja2 | 16 ++++++++++++++-- fietsboek/transformers/__init__.py | 34 ++++++++++++++++++++++------------ 2 files changed, 36 insertions(+), 14 deletions(-) diff --git a/fietsboek/templates/edit_form.jinja2 b/fietsboek/templates/edit_form.jinja2 index 8f6e092..1abe076 100644 --- a/fietsboek/templates/edit_form.jinja2 +++ b/fietsboek/templates/edit_form.jinja2 @@ -169,9 +169,9 @@ {% if params is not none %} - {{ transformer.parameter_model().parse_obj(params).html_ui("transformer[{}][%%]".format(transformer.identifier())) }} + {{ render_parameters(transformer.identifier(), transformer.parameter_model().parse_obj(params).html_ui()) }} {% else %} - {{ transformer().parameters.html_ui("transformer[{}][%%]".format(transformer.identifier())) }} + {{ render_parameters(transformer.identifier(), transformer().parameters.html_ui()) }} {% endif %}
@@ -179,3 +179,15 @@
{% endmacro %} + +{% macro render_parameters(identifier, params) %} +{% for param in params %} +{% if param.type == "int" %} + + +{% elif param.type == "str" %} + + +{% endif %} +{% endfor %} +{% endmacro %} diff --git a/fietsboek/transformers/__init__.py b/fietsboek/transformers/__init__.py index 1f084a0..f3afb57 100644 --- a/fietsboek/transformers/__init__.py +++ b/fietsboek/transformers/__init__.py @@ -12,10 +12,9 @@ function to load and apply transformers. from abc import ABC, abstractmethod from collections.abc import Mapping from itertools import chain, islice -from typing import TypeVar +from typing import Literal, NamedTuple, TypeVar from gpxpy.gpx import GPX -from markupsafe import Markup from pydantic import BaseModel from pyramid.i18n import TranslationString from pyramid.request import Request @@ -25,6 +24,25 @@ _ = TranslationString T = TypeVar("T", bound="Transformer") +class ParameterDefinition(NamedTuple): + """A parameter definition for the UI to render.""" + + type: Literal["int", "str"] + """Type of the parameter.""" + + name: str + """Name of the parameter. + + This is the machine-readable identifier, not the human readable name. + """ + + label: TranslationString + """Human-readable label of the parameter.""" + + value: str + """The serialized value of the parameter.""" + + class Parameters(BaseModel): """Parameters for a transformer. @@ -32,26 +50,18 @@ class Parameters(BaseModel): parameters to be serialized from and to POST request parameters. """ - def html_ui(self, name_template: str) -> Markup: + def html_ui(self) -> list[ParameterDefinition]: """Renders a HTML UI for this parameter set. - :param name_template: The template for the HTML form element names, - with a %% placeholder for the parameter name. :return: The rendered UI, ready for inclusion. """ - # TODO: Implement this based on the model's fields - # This is probably done better in the actual template, we shouldn't - # return Markup straight away. - # Also think of the localization. - return Markup() + return [] def read_from_request(self, data: Mapping[str, str]): """Parses the parameters from the given request data. - :param prefix: The prefix of the input parameter names. :param data: The request data, e.g. from the POST values. """ - # TODO: Implement parsing class Transformer(ABC): -- cgit v1.2.3 From 4df57a352460fe0944b73ddf738d678b772c9bd6 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Thu, 2 Mar 2023 18:22:59 +0100 Subject: fix lint --- fietsboek/actions.py | 2 +- pylint.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fietsboek/actions.py b/fietsboek/actions.py index 2058f4d..0adac94 100644 --- a/fietsboek/actions.py +++ b/fietsboek/actions.py @@ -64,7 +64,7 @@ def add_track( :return: The track object that has been inserted into the database. Useful for its ``id`` attribute. """ - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments,too-many-locals LOGGER.debug("Inserting new track...") track = models.Track( owner=owner, diff --git a/pylint.toml b/pylint.toml index 9e3a63c..f1e4270 100644 --- a/pylint.toml +++ b/pylint.toml @@ -17,7 +17,7 @@ # be loaded. Extensions are loading into the active Python interpreter and may # run arbitrary code. (This is an alternative name to extension-pkg-allow-list # for backward compatibility.) -# extension-pkg-whitelist = +extension-pkg-whitelist = ["pydantic"] # Return non-zero exit code if any of these messages/categories are detected, # even if score is above --fail-under value. Syntax same as enable. Messages -- cgit v1.2.3 From ac76ae25c5da1f8b3ab963cb0d4468026cc17afa Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Thu, 2 Mar 2023 21:23:12 +0100 Subject: FixNullElevation: also take into account slope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For some reason, I have GPX tracks that have the first two points be ~100 meters apart in elevation, but only ~20 meters apart in distance. This is quite unrealistic and produces pretty bad height plots (almost as bad as the zero elevation). Since the issue is very related, and the fix is pretty much the same, I thought it would be a good idea to adapt the FixNullElevation transformer to handle this case as well. For reference, "the internet" says that the maximum slope for a MTB is ~15% to ~35%, depending on the conditions - with 35% being pretty steep. I think it's fair to throw away elevations that exceed 100% (basically a 45° angle upwards), especially since we only discard them at the start and end. --- fietsboek/transformers/__init__.py | 66 ++++++++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 13 deletions(-) diff --git a/fietsboek/transformers/__init__.py b/fietsboek/transformers/__init__.py index f3afb57..f98a545 100644 --- a/fietsboek/transformers/__init__.py +++ b/fietsboek/transformers/__init__.py @@ -11,10 +11,10 @@ function to load and apply transformers. from abc import ABC, abstractmethod from collections.abc import Mapping -from itertools import chain, islice +from itertools import islice from typing import Literal, NamedTuple, TypeVar -from gpxpy.gpx import GPX +from gpxpy.gpx import GPX, GPXTrackPoint from pydantic import BaseModel from pyramid.i18n import TranslationString from pyramid.request import Request @@ -164,17 +164,57 @@ class FixNullElevation(Transformer): for point in segment.points ) - points = all_points() - previous_points = chain([None], all_points()) - next_points = chain(islice(all_points(), 1, None), [None]) - - for previous_point, point, next_point in zip(previous_points, points, next_points): - if point.elevation == 0.0: - if previous_point: - point.elevation += previous_point.elevation - if next_point: - point.elevation += next_point.elevation - point.elevation /= sum(1 for pt in [previous_point, next_point] if pt) + def rev_points(): + return ( + point + for track in reversed(gpx.tracks) + for segment in reversed(track.segments) + for point in reversed(segment.points) + ) + + max_slope = 1.0 + + # First, from the front, find the first point with non-zero elevation (or low enough slope) + bad_until = 0 + final_elevation = 0 + for i, (point, next_point) in enumerate(zip(all_points(), islice(all_points(), 1, None))): + if point.elevation != 0.0 and self.slope(point, next_point) < max_slope: + bad_until = i + final_elevation = point.elevation + break + + for point in islice(all_points(), None, bad_until): + point.elevation = final_elevation + + # Second, from the back + bad_until = 0 + final_elevation = 0 + for i, (point, prev_point) in enumerate(zip(rev_points(), islice(rev_points(), 1, None))): + if point.elevation != 0.0 and self.slope(point, prev_point) < max_slope: + bad_until = i + final_elevation = point.elevation + break + + for point in islice(rev_points(), None, bad_until): + point.elevation = final_elevation + + @staticmethod + def slope(point_a: GPXTrackPoint, point_b: GPXTrackPoint) -> float: + """Returns the slope between two GPX points. + + This is defined as delta_h / euclid_distance. + + :param point_a: First point. + :param point_b: Second point. + :return: The slope, as percentage. + """ + if point_a.elevation is None or point_b.elevation is None: + return 0.0 + delta_h = abs(point_a.elevation - point_b.elevation) + dist = point_a.distance_2d(point_b) + if dist == 0.0 or dist is None: + return 0.0 + return delta_h / dist def list_transformers() -> list[type[Transformer]]: -- cgit v1.2.3 From 63779a61151babf9dc60d55acc81e3a97811a60e Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Sat, 4 Mar 2023 17:43:02 +0100 Subject: transformers: de-duplicate fixup code --- fietsboek/transformers/__init__.py | 43 ++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/fietsboek/transformers/__init__.py b/fietsboek/transformers/__init__.py index f98a545..f6318d7 100644 --- a/fietsboek/transformers/__init__.py +++ b/fietsboek/transformers/__init__.py @@ -10,7 +10,7 @@ function to load and apply transformers. """ from abc import ABC, abstractmethod -from collections.abc import Mapping +from collections.abc import Callable, Iterable, Mapping from itertools import islice from typing import Literal, NamedTuple, TypeVar @@ -155,7 +155,7 @@ class FixNullElevation(Transformer): def parameters(self, value): pass - def execute(self, gpx): + def execute(self, gpx: GPX): def all_points(): return ( point @@ -172,30 +172,37 @@ class FixNullElevation(Transformer): for point in reversed(segment.points) ) - max_slope = 1.0 + # First, from the front + self.fixup(all_points) + # Then, from the back + self.fixup(rev_points) - # First, from the front, find the first point with non-zero elevation (or low enough slope) - bad_until = 0 - final_elevation = 0 - for i, (point, next_point) in enumerate(zip(all_points(), islice(all_points(), 1, None))): - if point.elevation != 0.0 and self.slope(point, next_point) < max_slope: - bad_until = i - final_elevation = point.elevation - break + @classmethod + def fixup(cls, points: Callable[[], Iterable[GPXTrackPoint]]): + """Fixes the given GPX points. - for point in islice(all_points(), None, bad_until): - point.elevation = final_elevation + This iterates over the points and checks for the first point that has a + non-zero elevation, and a slope that doesn't exceed 100%. All previous + points will have their elevation adjusted to match this first "good + point". + + :param points: A function that generates the iterable of points. + """ + max_slope = 1.0 - # Second, from the back bad_until = 0 - final_elevation = 0 - for i, (point, prev_point) in enumerate(zip(rev_points(), islice(rev_points(), 1, None))): - if point.elevation != 0.0 and self.slope(point, prev_point) < max_slope: + final_elevation = 0.0 + for i, (point, next_point) in enumerate(zip(points(), islice(points(), 1, None))): + if ( + point.elevation is not None + and point.elevation != 0.0 + and cls.slope(point, next_point) < max_slope + ): bad_until = i final_elevation = point.elevation break - for point in islice(rev_points(), None, bad_until): + for point in islice(points(), None, bad_until): point.elevation = final_elevation @staticmethod -- cgit v1.2.3 From e9025533b86228f7073a672269c939c5ae5db8e2 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Sat, 4 Mar 2023 18:24:11 +0100 Subject: add tests for the new transformer --- fietsboek/templates/details.jinja2 | 20 +++--- tests/assets/Synthetic_Steep_Slope.gpx.gz | Bin 0 -> 883 bytes tests/assets/Synthetic_Zero_Elevation.gpx.gz | Bin 0 -> 791 bytes tests/playwright/test_transformers.py | 87 +++++++++++++++++++++++++++ tests/testutils.py | 24 +++++++- 5 files changed, 118 insertions(+), 13 deletions(-) create mode 100644 tests/assets/Synthetic_Steep_Slope.gpx.gz create mode 100644 tests/assets/Synthetic_Zero_Elevation.gpx.gz create mode 100644 tests/playwright/test_transformers.py diff --git a/fietsboek/templates/details.jinja2 b/fietsboek/templates/details.jinja2 index 2ff492c..7c1ba8c 100644 --- a/fietsboek/templates/details.jinja2 +++ b/fietsboek/templates/details.jinja2 @@ -88,46 +88,46 @@ {{ _("page.details.date") }} - {{ track.date | format_datetime }} + {{ track.date | format_datetime }} {% if show_organic %} {{ _("page.details.start_time") }} - {{ track.start_time | format_datetime }} + {{ track.start_time | format_datetime }} {{ _("page.details.end_time") }} - {{ track.end_time | format_datetime }} + {{ track.end_time | format_datetime }} {% endif %} {{ _("page.details.length") }} - {{ (track.length / 1000) | round(2) | format_decimal }} km + {{ (track.length / 1000) | round(2) | format_decimal }} km {{ _("page.details.uphill") }} - {{ track.uphill | round(2) | format_decimal }} m + {{ track.uphill | round(2) | format_decimal }} m {{ _("page.details.downhill") }} - {{ track.downhill | round(2) | format_decimal }} m + {{ track.downhill | round(2) | format_decimal }} m {% if show_organic %} {{ _("page.details.moving_time") }} - {{ track.moving_time }} + {{ track.moving_time }} {{ _("page.details.stopped_time") }} - {{ track.stopped_time }} + {{ track.stopped_time }} {{ _("page.details.max_speed") }} - {{ mps_to_kph(track.max_speed) | round(2) | format_decimal }} km/h + {{ mps_to_kph(track.max_speed) | round(2) | format_decimal }} km/h {{ _("page.details.avg_speed") }} - {{ mps_to_kph(track.avg_speed) | round(2) | format_decimal }} km/h + {{ mps_to_kph(track.avg_speed) | round(2) | format_decimal }} km/h {% endif %} diff --git a/tests/assets/Synthetic_Steep_Slope.gpx.gz b/tests/assets/Synthetic_Steep_Slope.gpx.gz new file mode 100644 index 0000000..244f78a Binary files /dev/null and b/tests/assets/Synthetic_Steep_Slope.gpx.gz differ diff --git a/tests/assets/Synthetic_Zero_Elevation.gpx.gz b/tests/assets/Synthetic_Zero_Elevation.gpx.gz new file mode 100644 index 0000000..94abb0a Binary files /dev/null and b/tests/assets/Synthetic_Zero_Elevation.gpx.gz differ diff --git a/tests/playwright/test_transformers.py b/tests/playwright/test_transformers.py new file mode 100644 index 0000000..0fc4ffc --- /dev/null +++ b/tests/playwright/test_transformers.py @@ -0,0 +1,87 @@ +from playwright.sync_api import Page, expect +from sqlalchemy import select + +from testutils import extract_and_upload +from fietsboek import models + + +def test_transformer_zero_elevation_disabled(page: Page, playwright_helper, tmp_path, dbaccess): + playwright_helper.login() + + page.goto("/") + page.get_by_text("Upload").click() + + extract_and_upload(page, "Synthetic_Zero_Elevation.gpx.gz", tmp_path) + + page.locator(".btn", has_text="Upload").click() + + # Once we have finished the upload, extract the ID of the track and check + # the properties + new_track_id = int(page.url.rsplit("/", 1)[1]) + track = dbaccess.execute(select(models.Track).filter_by(id=new_track_id)).scalar_one() + + assert track.cache.uphill > 160 + expect(page.locator("#detailsUphill")).to_contain_text("167.7 m") + + +def test_transformer_zero_elevation_enabled(page: Page, playwright_helper, tmp_path, dbaccess): + playwright_helper.login() + + page.goto("/") + page.get_by_text("Upload").click() + + extract_and_upload(page, "Synthetic_Zero_Elevation.gpx.gz", tmp_path) + + page.locator("#transformer-heading-1 .accordion-button").click() + page.locator("#transformer-enabled-1").click() + + page.locator(".btn", has_text="Upload").click() + + # Once we have finished the upload, extract the ID of the track and check + # the properties + new_track_id = int(page.url.rsplit("/", 1)[1]) + track = dbaccess.execute(select(models.Track).filter_by(id=new_track_id)).scalar_one() + + assert track.cache.uphill < 0.1 + expect(page.locator("#detailsUphill")).to_contain_text("0 m") + + +def test_transformer_steep_slope_disabled(page: Page, playwright_helper, tmp_path, dbaccess): + playwright_helper.login() + + page.goto("/") + page.get_by_text("Upload").click() + + extract_and_upload(page, "Synthetic_Steep_Slope.gpx.gz", tmp_path) + + page.locator(".btn", has_text="Upload").click() + + # Once we have finished the upload, extract the ID of the track and check + # the properties + new_track_id = int(page.url.rsplit("/", 1)[1]) + track = dbaccess.execute(select(models.Track).filter_by(id=new_track_id)).scalar_one() + + assert track.cache.uphill > 60 + expect(page.locator("#detailsUphill")).to_contain_text("61.54 m") + + +def test_transformer_steep_slope_enabled(page: Page, playwright_helper, tmp_path, dbaccess): + playwright_helper.login() + + page.goto("/") + page.get_by_text("Upload").click() + + extract_and_upload(page, "Synthetic_Steep_Slope.gpx.gz", tmp_path) + + page.locator("#transformer-heading-1 .accordion-button").click() + page.locator("#transformer-enabled-1").click() + + page.locator(".btn", has_text="Upload").click() + + # Once we have finished the upload, extract the ID of the track and check + # the properties + new_track_id = int(page.url.rsplit("/", 1)[1]) + track = dbaccess.execute(select(models.Track).filter_by(id=new_track_id)).scalar_one() + + assert track.cache.uphill < 2 + expect(page.locator("#detailsUphill")).to_contain_text("1.2 m") diff --git a/tests/testutils.py b/tests/testutils.py index 3ddbdbe..810bdf7 100644 --- a/tests/testutils.py +++ b/tests/testutils.py @@ -2,19 +2,37 @@ import gzip from pathlib import Path +from playwright.sync_api import Page -def load_gpx_asset(filename): + +def load_gpx_asset(filename: str) -> bytes: """Load a GPX test asset. This looks in the tests/assets/ folder, reads and unzips the file and returns its contents. :param filename: Name of the asset to load. - :type filename: str :return: The content of the asset as bytes. - :rtype: bytes """ asset_dir = Path(__file__).parent / 'assets' test_file = asset_dir / filename with gzip.open(test_file, 'rb') as fobj: return fobj.read() + + +def extract_and_upload(page: Page, filename: str, tmp_path: Path): + """Extracts the given test asset, fills in the upload form and presses + upload. + + :param page: The playwright page on which to execute the actions. + :param filename: The filename. + :param tmp_path: The temporary path (as given by pytest). + """ + gpx_data = load_gpx_asset(filename) + gpx_path = tmp_path / "Upload.gpx" + with open(gpx_path, "wb") as gpx_fobj: + gpx_fobj.write(gpx_data) + + page.get_by_label("GPX file").set_input_files(gpx_path) + + page.locator(".bi-upload").click() -- cgit v1.2.3 From 5a7c780b4f171de2476a4ff060cc450b4e3a3749 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Tue, 7 Mar 2023 19:03:02 +0100 Subject: also test transformers when editing a track --- tests/playwright/test_transformers.py | 48 +++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/playwright/test_transformers.py b/tests/playwright/test_transformers.py index 0fc4ffc..d12be5e 100644 --- a/tests/playwright/test_transformers.py +++ b/tests/playwright/test_transformers.py @@ -46,6 +46,30 @@ def test_transformer_zero_elevation_enabled(page: Page, playwright_helper, tmp_p expect(page.locator("#detailsUphill")).to_contain_text("0 m") +def test_transformer_zero_elevation_edited(page: Page, playwright_helper, tmp_path, dbaccess): + playwright_helper.login() + + page.goto("/") + page.get_by_text("Upload").click() + + extract_and_upload(page, "Synthetic_Zero_Elevation.gpx.gz", tmp_path) + + page.locator(".btn", has_text="Upload").click() + + page.locator(".btn", has_text="Edit").click() + + page.locator("#transformer-heading-1 .accordion-button").click() + page.locator("#transformer-enabled-1").click() + + page.locator(".btn", has_text="Save").click() + + track_id = int(page.url.rsplit("/", 1)[1]) + track = dbaccess.execute(select(models.Track).filter_by(id=track_id)).scalar_one() + + assert track.cache.uphill < 0.1 + expect(page.locator("#detailsUphill")).to_contain_text("0 m") + + def test_transformer_steep_slope_disabled(page: Page, playwright_helper, tmp_path, dbaccess): playwright_helper.login() @@ -85,3 +109,27 @@ def test_transformer_steep_slope_enabled(page: Page, playwright_helper, tmp_path assert track.cache.uphill < 2 expect(page.locator("#detailsUphill")).to_contain_text("1.2 m") + + +def test_transformer_steep_slope_edited(page: Page, playwright_helper, tmp_path, dbaccess): + playwright_helper.login() + + page.goto("/") + page.get_by_text("Upload").click() + + extract_and_upload(page, "Synthetic_Steep_Slope.gpx.gz", tmp_path) + + page.locator(".btn", has_text="Upload").click() + + page.locator(".btn", has_text="Edit").click() + + page.locator("#transformer-heading-1 .accordion-button").click() + page.locator("#transformer-enabled-1").click() + + page.locator(".btn", has_text="Save").click() + + track_id = int(page.url.rsplit("/", 1)[1]) + track = dbaccess.execute(select(models.Track).filter_by(id=track_id)).scalar_one() + + assert track.cache.uphill < 2 + expect(page.locator("#detailsUphill")).to_contain_text("1.2 m") -- cgit v1.2.3 From d5f8c6cb6e5156e1e6ee29cf9c86f58db5a540a7 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Tue, 7 Mar 2023 19:19:40 +0100 Subject: add some documentation about transformers --- doc/index.rst | 1 + doc/user.rst | 12 ++++++++++ doc/user/images/fixed_elevation.png | Bin 0 -> 8658 bytes doc/user/images/wrong_elevation.png | Bin 0 -> 7824 bytes doc/user/transformers.rst | 45 ++++++++++++++++++++++++++++++++++++ 5 files changed, 58 insertions(+) create mode 100644 doc/user.rst create mode 100644 doc/user/images/fixed_elevation.png create mode 100644 doc/user/images/wrong_elevation.png create mode 100644 doc/user/transformers.rst diff --git a/doc/index.rst b/doc/index.rst index 9ce95b9..0b69ef1 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -12,6 +12,7 @@ Welcome to Fietsboek's documentation! administration developer + user .. toctree:: :maxdepth: 1 diff --git a/doc/user.rst b/doc/user.rst new file mode 100644 index 0000000..ab210dc --- /dev/null +++ b/doc/user.rst @@ -0,0 +1,12 @@ +User Guide +========== + +.. toctree:: + :maxdepth: 2 + :caption: Contents + + user/transformers + +This is the user guide for Fietsboek! In here, you can find information that +might be interesting if you plan on sharing your tracks on a Fietsboek +instance. diff --git a/doc/user/images/fixed_elevation.png b/doc/user/images/fixed_elevation.png new file mode 100644 index 0000000..10d34c7 Binary files /dev/null and b/doc/user/images/fixed_elevation.png differ diff --git a/doc/user/images/wrong_elevation.png b/doc/user/images/wrong_elevation.png new file mode 100644 index 0000000..2e8557b Binary files /dev/null and b/doc/user/images/wrong_elevation.png differ diff --git a/doc/user/transformers.rst b/doc/user/transformers.rst new file mode 100644 index 0000000..c8ddf01 --- /dev/null +++ b/doc/user/transformers.rst @@ -0,0 +1,45 @@ +Transformers +============ + +Transformers are small (or big) transformations that are applied to your GPX +track after it has been uploaded. This allows Fietsboek to provide some common, +simple editing options, without users having to do that themselves. + +All transformers are optional and disabled by default. You can enable +transformers when uploading a track on the bottom of the page. You can also +enable transformers for already uploaded tracks in the editing view. + +.. note:: + + When enabling transformers for already existing tracks, your browser might + not show the changed track. Try force-refreshing (Crtl + F5) the page so + that it reloads the GPX from the server. + +In other applications, transformers are sometimes called "filters". That term +however has many different meanings (like the filters on the "Browse" page), +and as such, Fietsboek calls them transformers. + +Fix Null Elevation +------------------ + +The *fix null elevation* transformer removes points at the start and end of a +track that have a "wrong" elevation. This helps to avoid issues when GPX +trackers don't have elevation data yet and fill in 0, leading to wrong uphill +calculations and wrong diagrams: + +.. image:: images/wrong_elevation.png + :width: 200 + :alt: An elevation graph that starts at 0 and makes a jump to 165. + +Activating the transformer will produce the following track: + +.. image:: images/fixed_elevation.png + :width: 200 + :alt: The same track, but with a fixed elevation graph that starts at 165. + +The transformer considers "wrong" elevation to be either points that have an +elevation of 0, or points that have a slope of more than 100% to the next +point. + +To fix those points, the transformer will find the first correct point, and +copy its elevation to the wrong points. -- cgit v1.2.3 From f232660e17f9dfe642f986ebb0de34a3c8f24c83 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Tue, 7 Mar 2023 19:49:29 +0100 Subject: fix typo in log message --- fietsboek/actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fietsboek/actions.py b/fietsboek/actions.py index 0adac94..16631b2 100644 --- a/fietsboek/actions.py +++ b/fietsboek/actions.py @@ -191,7 +191,7 @@ def execute_transformers(request: Request, track: models.Track) -> Optional[gpxp serialized = [[tfm.identifier(), tfm.parameters.dict()] for tfm in settings] if serialized == track.transformers: - LOGGER.debug("Applied transformations mach on %d, skipping", track.id) + LOGGER.debug("Applied transformations match on %d, skipping", track.id) return None # We always start with the backup, that way we don't get "deepfried GPX" -- cgit v1.2.3 From e8678a7155ff64d797693f7a8ec84c196d1d4748 Mon Sep 17 00:00:00 2001 From: Daniel Schadt Date: Tue, 7 Mar 2023 19:54:07 +0100 Subject: switch order of expect & assert While it shouldn't change the outcome of the test, it might make the test less flaky, as the expect call will wait until the page is loaded - which also indicates that the data is updated. Without this, the test depends on the backend being "fast enough" with applying the transformation. --- tests/playwright/test_transformers.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/tests/playwright/test_transformers.py b/tests/playwright/test_transformers.py index d12be5e..0b2e4de 100644 --- a/tests/playwright/test_transformers.py +++ b/tests/playwright/test_transformers.py @@ -15,13 +15,14 @@ def test_transformer_zero_elevation_disabled(page: Page, playwright_helper, tmp_ page.locator(".btn", has_text="Upload").click() - # Once we have finished the upload, extract the ID of the track and check - # the properties + # Expect early (here and in the other tests) to ensure that the backend has + # caught up with executing the transformer. Otherwise it might happen that + # we read the database while the request is not finished yet. + expect(page.locator("#detailsUphill")).to_contain_text("167.7 m") new_track_id = int(page.url.rsplit("/", 1)[1]) track = dbaccess.execute(select(models.Track).filter_by(id=new_track_id)).scalar_one() assert track.cache.uphill > 160 - expect(page.locator("#detailsUphill")).to_contain_text("167.7 m") def test_transformer_zero_elevation_enabled(page: Page, playwright_helper, tmp_path, dbaccess): @@ -37,13 +38,11 @@ def test_transformer_zero_elevation_enabled(page: Page, playwright_helper, tmp_p page.locator(".btn", has_text="Upload").click() - # Once we have finished the upload, extract the ID of the track and check - # the properties + expect(page.locator("#detailsUphill")).to_contain_text("0 m") new_track_id = int(page.url.rsplit("/", 1)[1]) track = dbaccess.execute(select(models.Track).filter_by(id=new_track_id)).scalar_one() assert track.cache.uphill < 0.1 - expect(page.locator("#detailsUphill")).to_contain_text("0 m") def test_transformer_zero_elevation_edited(page: Page, playwright_helper, tmp_path, dbaccess): @@ -63,11 +62,11 @@ def test_transformer_zero_elevation_edited(page: Page, playwright_helper, tmp_pa page.locator(".btn", has_text="Save").click() + expect(page.locator("#detailsUphill")).to_contain_text("0 m") track_id = int(page.url.rsplit("/", 1)[1]) track = dbaccess.execute(select(models.Track).filter_by(id=track_id)).scalar_one() assert track.cache.uphill < 0.1 - expect(page.locator("#detailsUphill")).to_contain_text("0 m") def test_transformer_steep_slope_disabled(page: Page, playwright_helper, tmp_path, dbaccess): @@ -80,13 +79,11 @@ def test_transformer_steep_slope_disabled(page: Page, playwright_helper, tmp_pat page.locator(".btn", has_text="Upload").click() - # Once we have finished the upload, extract the ID of the track and check - # the properties + expect(page.locator("#detailsUphill")).to_contain_text("61.54 m") new_track_id = int(page.url.rsplit("/", 1)[1]) track = dbaccess.execute(select(models.Track).filter_by(id=new_track_id)).scalar_one() assert track.cache.uphill > 60 - expect(page.locator("#detailsUphill")).to_contain_text("61.54 m") def test_transformer_steep_slope_enabled(page: Page, playwright_helper, tmp_path, dbaccess): @@ -102,13 +99,11 @@ def test_transformer_steep_slope_enabled(page: Page, playwright_helper, tmp_path page.locator(".btn", has_text="Upload").click() - # Once we have finished the upload, extract the ID of the track and check - # the properties + expect(page.locator("#detailsUphill")).to_contain_text("1.2 m") new_track_id = int(page.url.rsplit("/", 1)[1]) track = dbaccess.execute(select(models.Track).filter_by(id=new_track_id)).scalar_one() assert track.cache.uphill < 2 - expect(page.locator("#detailsUphill")).to_contain_text("1.2 m") def test_transformer_steep_slope_edited(page: Page, playwright_helper, tmp_path, dbaccess): @@ -128,8 +123,8 @@ def test_transformer_steep_slope_edited(page: Page, playwright_helper, tmp_path, page.locator(".btn", has_text="Save").click() + expect(page.locator("#detailsUphill")).to_contain_text("1.2 m") track_id = int(page.url.rsplit("/", 1)[1]) track = dbaccess.execute(select(models.Track).filter_by(id=track_id)).scalar_one() assert track.cache.uphill < 2 - expect(page.locator("#detailsUphill")).to_contain_text("1.2 m") -- cgit v1.2.3