diff options
| author | Daniel Schadt <kingdread@gmx.de> | 2026-10-17 12:42:32 +0200 |
|---|---|---|
| committer | Daniel Schadt <kingdread@gmx.de> | 2026-10-17 12:42:32 +0200 |
| commit | b8504a9d5bf6d58bc1ca00bedccb2eaef1ab5085 (patch) | |
| tree | 8dc6b6a69f8adc2ea5f41ae54044f6a2602b14c8 | |
| parent | be568d5386e550f2b2ba4fbfc73cb7a45f605506 (diff) | |
| parent | 39ac4a457593f94438efc3cacff411b93ba53324 (diff) | |
| download | fietsboek-b8504a9d5bf6d58bc1ca00bedccb2eaef1ab5085.tar.gz fietsboek-b8504a9d5bf6d58bc1ca00bedccb2eaef1ab5085.tar.bz2 fietsboek-b8504a9d5bf6d58bc1ca00bedccb2eaef1ab5085.zip | |
Merge remote-tracking branch 'origin/smooth-elevation'
| -rw-r--r-- | fietsboek/actions.py | 20 | ||||
| -rw-r--r-- | fietsboek/locale/de/LC_MESSAGES/messages.mo | bin | 21619 -> 21820 bytes | |||
| -rw-r--r-- | fietsboek/locale/de/LC_MESSAGES/messages.po | 31 | ||||
| -rw-r--r-- | fietsboek/locale/en/LC_MESSAGES/messages.mo | bin | 20345 -> 20553 bytes | |||
| -rw-r--r-- | fietsboek/locale/en/LC_MESSAGES/messages.po | 31 | ||||
| -rw-r--r-- | fietsboek/locale/fietslog.pot | 26 | ||||
| -rw-r--r-- | fietsboek/models/track.py | 42 | ||||
| -rw-r--r-- | fietsboek/templates/edit.jinja2 | 2 | ||||
| -rw-r--r-- | fietsboek/templates/edit_form.jinja2 | 5 | ||||
| -rw-r--r-- | fietsboek/templates/finish_upload.jinja2 | 2 | ||||
| -rw-r--r-- | fietsboek/transformers/__init__.py | 20 | ||||
| -rw-r--r-- | fietsboek/transformers/breaks.py | 4 | ||||
| -rw-r--r-- | fietsboek/transformers/elevation.py | 128 | ||||
| -rw-r--r-- | fietsboek/updater/scripts/upd_20261015_gg7gfmpk4nxl062z.py | 69 | ||||
| -rw-r--r-- | poetry.lock | 16 | ||||
| -rw-r--r-- | pyproject.toml | 1 | ||||
| -rw-r--r-- | tests/playwright/test_transformers.py | 38 |
17 files changed, 360 insertions, 75 deletions
diff --git a/fietsboek/actions.py b/fietsboek/actions.py index b81b56f..29d7043 100644 --- a/fietsboek/actions.py +++ b/fietsboek/actions.py @@ -216,27 +216,11 @@ def execute_transformers(request: Request, track: models.Track): LOGGER.debug("Applied transformations match on %d, skipping", track.id) return - # We always start with the backup, that way we don't get "deepfried GPX" - # files by having the same filters run multiple times on the same input. - # They are not idempotent after all. - manager = request.data_manager.open(track.id) - backup_bytes = manager.decompress_backup() - reloaded = convert.smart_convert(backup_bytes) - path = reloaded.path() - - for transformer in settings: - LOGGER.debug("Running %s with %r", transformer, transformer.parameters) - transformer.execute(path) - - track.fast_set_path(path) - LOGGER.debug("Saving new transformers on %d", track.id) track.transformers = serialized - LOGGER.debug("Rebuilding cache for %d", track.id) - request.dbsession.delete(track.cache) - track.cache = None - track.ensure_cache() + track.execute_transformers(request.data_manager) + request.dbsession.add(track.cache) diff --git a/fietsboek/locale/de/LC_MESSAGES/messages.mo b/fietsboek/locale/de/LC_MESSAGES/messages.mo Binary files differindex 2cd5007..08a51ec 100644 --- a/fietsboek/locale/de/LC_MESSAGES/messages.mo +++ b/fietsboek/locale/de/LC_MESSAGES/messages.mo diff --git a/fietsboek/locale/de/LC_MESSAGES/messages.po b/fietsboek/locale/de/LC_MESSAGES/messages.po index 552880f..2fd2aad 100644 --- a/fietsboek/locale/de/LC_MESSAGES/messages.po +++ b/fietsboek/locale/de/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2026-10-01 13:36+0200\n" +"POT-Creation-Date: 2026-10-14 19:07+0200\n" "PO-Revision-Date: 2022-07-02 17:35+0200\n" "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" "Language: de\n" @@ -18,11 +18,11 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 2.18.0\n" -#: fietsboek/actions.py:269 +#: fietsboek/actions.py:271 msgid "email.verify_mail.subject" msgstr "Fietsboek Konto Bestätigung" -#: fietsboek/actions.py:272 +#: fietsboek/actions.py:274 msgid "email.verify.text" msgstr "" "Um Dein Fietsboek-Konto zu bestätigen, nutze diesen Link: {}\n" @@ -551,8 +551,8 @@ msgstr "Schlagwörter" #: fietsboek/templates/details.jinja2:111 msgid "page.details.background_tasks_in_progress" msgstr "" -"Hintergrundaufgaben für diese Strecke sind noch nicht abgeschlossen. " -"Die angezeigten Informationen sind noch unvollständig." +"Hintergrundaufgaben für diese Strecke sind noch nicht abgeschlossen. Die " +"angezeigten Informationen sind noch unvollständig." #: fietsboek/templates/details.jinja2:121 fietsboek/templates/edit.jinja2:10 #: fietsboek/templates/finish_upload.jinja2:10 @@ -733,7 +733,7 @@ msgstr "Bildbeschreibung" msgid "page.track.form.image_description_modal.save" msgstr "Übernehmen" -#: fietsboek/templates/edit_form.jinja2:175 +#: fietsboek/templates/edit_form.jinja2:176 msgid "page.track.form.transformer.enable" msgstr "Transformation anwenden" @@ -1218,26 +1218,35 @@ msgstr "Pausen entfernen" msgid "transformers.remove-breaks.description" msgstr "Diese Transformation entfernt längere Pausen aus der Aufnahme." -#: fietsboek/transformers/elevation.py:43 +#: fietsboek/transformers/elevation.py:53 msgid "transformers.fix-null-elevation.title" msgstr "Nullhöhen beheben" -#: fietsboek/transformers/elevation.py:47 +#: fietsboek/transformers/elevation.py:57 msgid "transformers.fix-null-elevation.description" msgstr "" "Diese Transformation passt die Höhenangabe für Punkte an, bei denen die " "Höhenangabe fehlt." -#: fietsboek/transformers/elevation.py:109 +#: fietsboek/transformers/elevation.py:123 msgid "transformers.fix-elevation-jumps" msgstr "Höhensprünge beheben" -#: fietsboek/transformers/elevation.py:113 +#: fietsboek/transformers/elevation.py:127 msgid "transformers.fix-elevation-jumps.description" msgstr "" "Diese Transformation passt die Höhenangabe für Punkte an, bei denen die " "Höhe sprunghaft steigt oder fällt." +#: fietsboek/transformers/elevation.py:199 +msgid "transformers.smooth-elevation" +msgstr "Höhenprofil glätten" + +#: fietsboek/transformers/elevation.py:203 +msgid "transformers.smooth-elevation.description" +msgstr "" +"Diese Transformation glättet das Höhenprofil, um Rauschen zu verringern." + #: fietsboek/views/account.py:54 msgid "flash.invalid_name" msgstr "Ungültiger Name" @@ -1338,7 +1347,7 @@ msgstr "Keine Datei ausgewählt" msgid "flash.upload_success" msgstr "Hochladen erfolgreich" -#: fietsboek/views/upload.py:202 +#: fietsboek/views/upload.py:203 msgid "flash.upload_cancelled" msgstr "Hochladen abgebrochen" diff --git a/fietsboek/locale/en/LC_MESSAGES/messages.mo b/fietsboek/locale/en/LC_MESSAGES/messages.mo Binary files differindex 19b3c28..a5dbb81 100644 --- a/fietsboek/locale/en/LC_MESSAGES/messages.mo +++ b/fietsboek/locale/en/LC_MESSAGES/messages.mo diff --git a/fietsboek/locale/en/LC_MESSAGES/messages.po b/fietsboek/locale/en/LC_MESSAGES/messages.po index 1feb09a..880da4b 100644 --- a/fietsboek/locale/en/LC_MESSAGES/messages.po +++ b/fietsboek/locale/en/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2026-10-01 13:36+0200\n" +"POT-Creation-Date: 2026-10-14 19:07+0200\n" "PO-Revision-Date: 2023-04-03 20:42+0200\n" "Last-Translator: \n" "Language: en\n" @@ -18,11 +18,11 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 2.18.0\n" -#: fietsboek/actions.py:269 +#: fietsboek/actions.py:271 msgid "email.verify_mail.subject" msgstr "Fietsboek Account Verification" -#: fietsboek/actions.py:272 +#: fietsboek/actions.py:274 msgid "email.verify.text" msgstr "" "To verify your Fietsboek account, please use this link: {}\n" @@ -549,8 +549,8 @@ msgstr "Tagged as" #: fietsboek/templates/details.jinja2:111 msgid "page.details.background_tasks_in_progress" msgstr "" -"Background tasks for this track are still running. " -"The information displayed is still incomplete." +"Background tasks for this track are still running. The information " +"displayed is still incomplete." #: fietsboek/templates/details.jinja2:121 fietsboek/templates/edit.jinja2:10 #: fietsboek/templates/finish_upload.jinja2:10 @@ -729,7 +729,7 @@ msgstr "Image description" msgid "page.track.form.image_description_modal.save" msgstr "Apply" -#: fietsboek/templates/edit_form.jinja2:175 +#: fietsboek/templates/edit_form.jinja2:176 msgid "page.track.form.transformer.enable" msgstr "Apply transformation" @@ -1212,22 +1212,31 @@ msgstr "Remove breaks" msgid "transformers.remove-breaks.description" msgstr "This transformer removes long breaks from the recording" -#: fietsboek/transformers/elevation.py:43 +#: fietsboek/transformers/elevation.py:53 msgid "transformers.fix-null-elevation.title" msgstr "Fix null elevation" -#: fietsboek/transformers/elevation.py:47 +#: fietsboek/transformers/elevation.py:57 msgid "transformers.fix-null-elevation.description" msgstr "This transformer fixes the elevation of points whose elevation is unset." -#: fietsboek/transformers/elevation.py:109 +#: fietsboek/transformers/elevation.py:123 msgid "transformers.fix-elevation-jumps" msgstr "Fix elevation jumps" -#: fietsboek/transformers/elevation.py:113 +#: fietsboek/transformers/elevation.py:127 msgid "transformers.fix-elevation-jumps.description" msgstr "This transformer fixes abrupt jumps in the elevation value." +#: fietsboek/transformers/elevation.py:199 +msgid "transformers.smooth-elevation" +msgstr "Smooth elevation profile" + +#: fietsboek/transformers/elevation.py:203 +msgid "transformers.smooth-elevation.description" +msgstr "" +"This transformer applies a smoothing to the elevation profile to reduce noise." + #: fietsboek/views/account.py:54 msgid "flash.invalid_name" msgstr "Invalid name" @@ -1327,7 +1336,7 @@ msgstr "No file selected" msgid "flash.upload_success" msgstr "Upload successful" -#: fietsboek/views/upload.py:202 +#: fietsboek/views/upload.py:203 msgid "flash.upload_cancelled" msgstr "Upload cancelled" diff --git a/fietsboek/locale/fietslog.pot b/fietsboek/locale/fietslog.pot index 4a9e203..0a066b8 100644 --- a/fietsboek/locale/fietslog.pot +++ b/fietsboek/locale/fietslog.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2026-10-01 13:36+0200\n" +"POT-Creation-Date: 2026-10-14 19:07+0200\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" "Language-Team: LANGUAGE <LL@li.org>\n" @@ -17,11 +17,11 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 2.18.0\n" -#: fietsboek/actions.py:269 +#: fietsboek/actions.py:271 msgid "email.verify_mail.subject" msgstr "" -#: fietsboek/actions.py:272 +#: fietsboek/actions.py:274 msgid "email.verify.text" msgstr "" @@ -719,7 +719,7 @@ msgstr "" msgid "page.track.form.image_description_modal.save" msgstr "" -#: fietsboek/templates/edit_form.jinja2:175 +#: fietsboek/templates/edit_form.jinja2:176 msgid "page.track.form.transformer.enable" msgstr "" @@ -1196,22 +1196,30 @@ msgstr "" msgid "transformers.remove-breaks.description" msgstr "" -#: fietsboek/transformers/elevation.py:43 +#: fietsboek/transformers/elevation.py:53 msgid "transformers.fix-null-elevation.title" msgstr "" -#: fietsboek/transformers/elevation.py:47 +#: fietsboek/transformers/elevation.py:57 msgid "transformers.fix-null-elevation.description" msgstr "" -#: fietsboek/transformers/elevation.py:109 +#: fietsboek/transformers/elevation.py:123 msgid "transformers.fix-elevation-jumps" msgstr "" -#: fietsboek/transformers/elevation.py:113 +#: fietsboek/transformers/elevation.py:127 msgid "transformers.fix-elevation-jumps.description" msgstr "" +#: fietsboek/transformers/elevation.py:199 +msgid "transformers.smooth-elevation" +msgstr "" + +#: fietsboek/transformers/elevation.py:203 +msgid "transformers.smooth-elevation.description" +msgstr "" + #: fietsboek/views/account.py:54 msgid "flash.invalid_name" msgstr "" @@ -1308,7 +1316,7 @@ msgstr "" msgid "flash.upload_success" msgstr "" -#: fietsboek/views/upload.py:202 +#: fietsboek/views/upload.py:203 msgid "flash.upload_cancelled" msgstr "" diff --git a/fietsboek/models/track.py b/fietsboek/models/track.py index 361cc12..13787c8 100644 --- a/fietsboek/models/track.py +++ b/fietsboek/models/track.py @@ -54,6 +54,7 @@ from sqlalchemy import ( from sqlalchemy.orm import Mapped, mapped_column, relationship from .. import geo, util +from ..data import DataManager from .meta import Base if TYPE_CHECKING: @@ -618,7 +619,7 @@ class Track(Base): if not self.transformers: return None - if not isinstance(self.transformers, dict): + if not isinstance(self.transformers, list): return None for t_id, settings in self.transformers: @@ -626,6 +627,45 @@ class Track(Base): return settings return None + def execute_transformers(self, data_manager: DataManager): + """Makes sure that the transformers applied to this track match what they should be. + + The track's data is loaded fresh from the backup (as given by the data manager). + """ + # The conversion returns a Track, so it imports this model. + # pylint: disable=import-outside-toplevel,cyclic-import + from .. import convert, transformers + + if self.transformers is None: + return + + manager = data_manager.open(self.id) + backup_bytes = manager.decompress_backup() + reloaded = convert.smart_convert(backup_bytes) + path = reloaded.path() + + if not isinstance(self.transformers, list): + raise ValueError("Track.transformers should be a list or None") + + for transformer_id, parameters in self.transformers: + if not isinstance(parameters, dict): + LOGGER.debug("Invalid parameters on track %s: %r", self.id, parameters) + continue + transformer_cls = transformers.get_transformer(transformer_id) + if transformer_cls is None: + LOGGER.info("Unknown transformer set in track %s: %s", self.id, transformer_id) + continue + transformer = transformer_cls() + transformer.parameters = transformer.parameter_model()(**parameters) + LOGGER.debug("Running %s with %r", transformer, transformer.parameters) + transformer.execute(path) + + self.fast_set_path(path) + + LOGGER.debug("Rebuilding cache for %d", self.id) + self.cache = None + self.ensure_cache() + class TrackWithMetadata: """A class to add metadata to a :class:`Track`. diff --git a/fietsboek/templates/edit.jinja2 b/fietsboek/templates/edit.jinja2 index 26c520f..a195082 100644 --- a/fietsboek/templates/edit.jinja2 +++ b/fietsboek/templates/edit.jinja2 @@ -14,7 +14,7 @@ <label class="form-label ui-element" for="newGpx">{{ _("page.edit.form.new_track") }}</label> <input id="newGpx" name="gpx" type="file" class="form-control ui-element"> </div> - {{ edit_form.edit_track(track.title, track.date_raw, track.date_tz or 0, track.visibility, track.type, track.description, track.text_tags(), badges, track.tagged_people, images) }} + {{ edit_form.edit_track(track.title, track.date_raw, track.date_tz or 0, track.visibility, track.type, track.description, track.text_tags(), badges, track.tagged_people, images, False) }} {{ util.hidden_csrf_input() }} <div class="btn-group" role="group"> <button type="submit" class="btn btn-primary ui-element"><i class="bi bi-save"></i> {{ _("page.edit.form.submit") }}</button> diff --git a/fietsboek/templates/edit_form.jinja2 b/fietsboek/templates/edit_form.jinja2 index e515c3b..371accc 100644 --- a/fietsboek/templates/edit_form.jinja2 +++ b/fietsboek/templates/edit_form.jinja2 @@ -1,4 +1,4 @@ -{% macro edit_track(title, date, date_tz, visibility, type, description, tags, badges, friends, images) %} +{% macro edit_track(title, date, date_tz, visibility, type, description, tags, badges, friends, images, enable_default_transformers) %} <div class="mb-3"> <label for="formTitle" class="form-label">{{ _("page.track.form.title") }}</label> <input class="form-control ui-element" type="text" id="formTitle" name="title" value="{{ title | default("", true) }}"> @@ -165,12 +165,13 @@ <div id="transformer-{{ loop.index}}" class="accordion-collapse collapse" aria-labelledby="transformer-heading-{{ loop.index }}"> <div class="accordion-body"> {% set params = track.transformer_params_for(transformer.identifier()) if track else None %} + {% set enabled = params is not none or (enable_default_transformers and transformer.enabled_by_default()) %} <!-- Short transformer description --> <p>{{ _(transformer.description()) }}</p> <!-- Checkbox to enable the transformer --> <div class="form-check"> - <input class="form-check-input" type="checkbox" value="on" id="transformer-enabled-{{ loop.index }}" name="transformer[{{ transformer.identifier() }}]"{% if params is not none %} checked{% endif %}> + <input class="form-check-input" type="checkbox" value="on" id="transformer-enabled-{{ loop.index }}" name="transformer[{{ transformer.identifier() }}]"{% if enabled %} checked{% endif %}> <label class="form-check-label" for="transformer-enabled-{{ loop.index }}"> {{ _("page.track.form.transformer.enable") }} </label> diff --git a/fietsboek/templates/finish_upload.jinja2 b/fietsboek/templates/finish_upload.jinja2 index 34d4aac..b8491ec 100644 --- a/fietsboek/templates/finish_upload.jinja2 +++ b/fietsboek/templates/finish_upload.jinja2 @@ -10,7 +10,7 @@ <noscript><p>{{ _("page.noscript") }}<p></noscript> </div> <form method="POST" enctype="multipart/form-data"> - {{ edit_form.edit_track(upload_title, upload_date, upload_date_tz, upload_visibility, upload_type, upload_description, upload_tags, badges, upload_tagged_people, []) }} + {{ edit_form.edit_track(upload_title, upload_date, upload_date_tz, upload_visibility, upload_type, upload_description, upload_tags, badges, upload_tagged_people, [], True) }} {{ util.hidden_csrf_input() }} <div class="btn-group" role="group"> <button type="submit" class="btn btn-primary ui-element">{{ _("page.upload.form.submit") }}</button> diff --git a/fietsboek/transformers/__init__.py b/fietsboek/transformers/__init__.py index 097fbaf..4542721 100644 --- a/fietsboek/transformers/__init__.py +++ b/fietsboek/transformers/__init__.py @@ -95,6 +95,11 @@ class Transformer(ABC): @classmethod @abstractmethod + def enabled_by_default(cls) -> bool: + """Whether this transformer should be enabled by default.""" + + @classmethod + @abstractmethod def parameter_model(cls: type[T]) -> type[Parameters]: """Returns the parameter model that this transformer expects. @@ -135,15 +140,28 @@ def list_transformers() -> list[type[Transformer]]: """ # pylint: disable=import-outside-toplevel,cyclic-import from .breaks import RemoveBreaks - from .elevation import FixElevationJumps, FixNullElevation + from .elevation import FixElevationJumps, FixNullElevation, SmoothElevation return [ FixNullElevation, FixElevationJumps, RemoveBreaks, + SmoothElevation, ] +def get_transformer(identifier: str) -> type[Transformer] | None: + """Retrieve the transformer with the given identifier. + + :param identifier: The identifier of the transformer to retrieve. + :return: The class of the transformer. + """ + for transformer in list_transformers(): + if transformer.identifier() == identifier: + return transformer + return None + + def extract_from_request(request: Request) -> list[Transformer]: """Extracts the list of transformers to execute from the given request. diff --git a/fietsboek/transformers/breaks.py b/fietsboek/transformers/breaks.py index 1072eef..81aca5a 100644 --- a/fietsboek/transformers/breaks.py +++ b/fietsboek/transformers/breaks.py @@ -36,6 +36,10 @@ class RemoveBreaks(Transformer): return _("transformers.remove-breaks.description") @classmethod + def enabled_by_default(cls) -> bool: + return False + + @classmethod def parameter_model(cls) -> type[Parameters]: return Parameters diff --git a/fietsboek/transformers/elevation.py b/fietsboek/transformers/elevation.py index 27683bb..ad44340 100644 --- a/fietsboek/transformers/elevation.py +++ b/fietsboek/transformers/elevation.py @@ -1,16 +1,26 @@ """Transformers that deal with elevation changes in the track.""" +import logging +import statistics from collections.abc import Callable, Iterable from itertools import islice, zip_longest from pyramid.i18n import TranslationString +from SGCC import savgol # type: ignore from .. import geo from . import Parameters, Transformer _ = TranslationString +LOGGER = logging.getLogger(__name__) MAX_ORGANIC_SLOPE: float = 1.0 +ELEVATION_SMOOTHING_WINDOW_LENGTH: float = 500.0 +"""One-sided window length (in m) of the elevation smoothing filter. + +The filtering will estimate how many points need to be included to reach this +length. +""" def slope(point_a: geo.Point, point_b: geo.Point) -> float: @@ -47,6 +57,10 @@ class FixNullElevation(Transformer): return _("transformers.fix-null-elevation.description") @classmethod + def enabled_by_default(cls) -> bool: + return True + + @classmethod def parameter_model(cls) -> type[Parameters]: return Parameters @@ -113,6 +127,10 @@ class FixElevationJumps(Transformer): return _("transformers.fix-elevation-jumps.description") @classmethod + def enabled_by_default(cls) -> bool: + return False + + @classmethod def parameter_model(cls) -> type[Parameters]: return Parameters @@ -137,4 +155,112 @@ class FixElevationJumps(Transformer): current_point.elevation += point_adjustment -__all__ = ["FixNullElevation", "FixElevationJumps"] +class SmoothElevation(Transformer): + """A transformer that smooths the elevation. + + **Technical description:** + + This transformer has the goal of smoothing out noise in the GPS elevation + recording, while keeping the overall elevation profile true to the original. + We do this in two parts: First, we estimate the noisiness of the input data. + Then we apply a `Savitzky-Golay`_ filter, whose strength is scaled by the + noisiness estimated in the first step. + + .. _Savitzky-Golay: https://en.wikipedia.org/wiki/Savitzky%E2%80%93Golay_filter + + **Step 1:** Noise estimation + + To estimate the noisiness, we walk through the file in windows of 5 points. + We do a linear regression through those points, and compute the sum of + absolute errors. + + The reason behind this is that we assume that the profile given by 5 points + can be approximated reasonably with a straight line, so deviations from this + line are likely caused by noise in the recording. + + We average the sum-of-errors over all windows in the track to reach the + track's total noisiness, this prevents outlier windows from affecting the + noisiness too much. + + **Step 2:** Filter + + The actual smoothing is done with a pass of the Savitzky-Golay filter. The + window size :math:`w` is scaled linearly according to the noisiness + :math:`w` determined in step 1 as :math:`w = n \\cdot f`. The factor + :math:`f` is fixed and determined manually with some example tracks. + """ + + @classmethod + def identifier(cls) -> str: + return "smooth-elevation" + + @classmethod + def name(cls) -> TranslationString: + return _("transformers.smooth-elevation") + + @classmethod + def description(cls) -> TranslationString: + return _("transformers.smooth-elevation.description") + + @classmethod + def enabled_by_default(cls) -> bool: + return True + + @classmethod + def parameter_model(cls) -> type[Parameters]: + return Parameters + + @property + def parameters(self) -> Parameters: + return Parameters() + + @parameters.setter + def parameters(self, value): + pass + + def execute(self, path: geo.Path): + radius = self._estimate_window_size(path) + radius = min(radius, int(len(path.points) / 2)) + LOGGER.debug("Savitzky-Golay radius: %s", radius) + if radius: + self._savitzky_golay(path, radius) + + def _snr(self, path: geo.Path) -> float: + window_size = 5 + errors: list[float] = [] + for start in range(0, len(path.points) - window_size): + elevations = [point.elevation for point in path.points[start : start + window_size]] + xs = list(range(len(elevations))) + + sl, intercept = statistics.linear_regression(xs, elevations) + errors.extend( + abs(elevation - (x * sl + intercept)) for x, elevation in enumerate(elevations) + ) + return statistics.mean(errors) + + def _estimate_window_size(self, path: geo.Path) -> int: + points_per_meter = len(path.points) / path.movement_data().length + snr = self._snr(path) + window_length = 6 * ELEVATION_SMOOTHING_WINDOW_LENGTH * snr + LOGGER.debug("signal-to-noise=%s, points_per_meter=%s", snr, points_per_meter) + return int(points_per_meter * window_length) + + def _savitzky_golay(self, path: geo.Path, radius: int): + elevations = [0.0] * len(path.points) + coeffs = savgol.get_coefficients(0, 2, 2 * radius + 1, 0) + + for i in range(0, radius + 1): + elevations[i] = path.points[i].elevation + elevations[-i - 1] = path.points[-i - 1].elevation + + for i in range(radius, len(path.points) - radius - 1): + points = path.points[i - radius : i + radius + 1] + assert len(points) == len(coeffs) + smoothed = sum(coef * point.elevation for coef, point in zip(coeffs, points)) + elevations[i] = smoothed + + for i, elevation in enumerate(elevations): + path.points[i].elevation = elevation + + +__all__ = ["FixNullElevation", "FixElevationJumps", "SmoothElevation"] diff --git a/fietsboek/updater/scripts/upd_20261015_gg7gfmpk4nxl062z.py b/fietsboek/updater/scripts/upd_20261015_gg7gfmpk4nxl062z.py new file mode 100644 index 0000000..4168c5d --- /dev/null +++ b/fietsboek/updater/scripts/upd_20261015_gg7gfmpk4nxl062z.py @@ -0,0 +1,69 @@ +"""Revision upgrade script gg7gfmpk4nxl062z + +Active the "smooth elevation" transformer. + +Date created: 2026-10-15 14:22:06.047695 +""" +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.orm import Session +from sqlalchemy.orm.attributes import flag_modified + +from fietsboek import models +from fietsboek.data import DataManager +from fietsboek.updater.script import UpdateScript + +update_id = 'gg7gfmpk4nxl062z' +previous = [ + '7fa8kflikvnlvuqe', +] +alembic_revision = 'b73ed8b45f67' + + +class Up(UpdateScript): + def pre_alembic(self, config): + pass + + def post_alembic(self, config): + data_manager = DataManager(Path(config["fietsboek.data_dir"])) + engine = self.dbengine(config) + session = Session(engine) + + with Session(engine) as session: + query = select(models.Track) + for track in session.execute(query).scalars(): + if track.type != models.track.TrackType.ORGANIC: + continue + self.tell(f"Applying smoothing to track {track.id}") + track.transformers.append(["smooth-elevation", {}]) + flag_modified(track, "transformers") + track.execute_transformers(data_manager) + session.add(track) + session.add(track.cache) + session.commit() + + +class Down(UpdateScript): + def pre_alembic(self, config): + pass + + def post_alembic(self, config): + data_manager = DataManager(Path(config["fietsboek.data_dir"])) + engine = self.dbengine(config) + session = Session(engine) + + with Session(engine) as session: + query = select(models.Track) + for track in session.execute(query).scalars(): + if track.type != models.track.TrackType.ORGANIC: + continue + self.tell(f"Removing smoothing from track {track.id}") + unsmoothed = [[k, v] for k, v in track.transformers if k != "smooth-elevation"] + if unsmoothed == track.transformers: + continue + track.transformers = unsmoothed + track.execute_transformers(data_manager) + session.add(track) + session.add(track.cache) + session.commit() diff --git a/poetry.lock b/poetry.lock index 3fc3a7a..251bc18 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.3.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.1 and should not be changed by hand. [[package]] name = "alabaster" @@ -2706,6 +2706,18 @@ files = [ ] [[package]] +name = "savgol-calculator" +version = "1.0.5" +description = "Python implementation of a coefficients calculator for the Savitsky-Golay filter with any parameters." +optional = false +python-versions = ">=3.6.0" +groups = ["main"] +files = [ + {file = "savgol-calculator-1.0.5.tar.gz", hash = "sha256:f0e49fdc1944f1aab3fda644d0b1d2fa9f6cd8f2e204599c0cf00c59234fbed3"}, + {file = "savgol_calculator-1.0.5-py2.py3-none-any.whl", hash = "sha256:dd0db764ea9322b593f04e8bafbed383a205ae125ef4330f795e30aceaa69c7e"}, +] + +[[package]] name = "setuptools" version = "82.0.1" description = "Most extensible Python build backend with support for C/C++ extension modules" @@ -3489,4 +3501,4 @@ test = ["zope.testing"] [metadata] lock-version = "2.1" python-versions = ">=3.11, <4" -content-hash = "b4d90215cb83febb48f381f7eb9d8d2aa4fe159a34bfb93b96fbef54ea943881" +content-hash = "1fd62aecf611f739cc79de3ea6a6f69665ca822960e9bde75954c8d255764ac3" diff --git a/pyproject.toml b/pyproject.toml index d3d22a1..e4c4347 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ dependencies = [ "matplotlib (>=3.10.7,<4.0.0)", "hittekaart-py @ git+https://gitlab.com/dunj3/hittekaart@9a576a8d3a8cb1d143be0ade0310f71b490cacb9#subdirectory=hittekaart-py", + "savgol-calculator (==1.0.5)", ] [project.urls] diff --git a/tests/playwright/test_transformers.py b/tests/playwright/test_transformers.py index d4e3456..cc897db 100644 --- a/tests/playwright/test_transformers.py +++ b/tests/playwright/test_transformers.py @@ -13,6 +13,17 @@ def disable_cache(page: Page): page.route("**", lambda route: route.continue_()) +def set_transformers(page: Page, states: list[bool]): + """Enable/disable the transformers according to the given states.""" + for i, state in enumerate(states, start=1): + page.locator(f"#transformer-heading-{i} .accordion-button").click() + page.locator(f"#transformer-{i}.collapse.show").wait_for() + if state: + page.locator(f"#transformer-enabled-{i}").check() + else: + page.locator(f"#transformer-enabled-{i}").uncheck() + + def test_transformer_zero_elevation_disabled(page: Page, playwright_helper, tmp_path, dbaccess): playwright_helper.login() @@ -21,6 +32,8 @@ def test_transformer_zero_elevation_disabled(page: Page, playwright_helper, tmp_ extract_and_upload(page, "Synthetic_Zero_Elevation.gpx.gz", tmp_path) + set_transformers(page, [False, False, False, False]) + page.locator(".btn", has_text="Upload").click() # Expect early (here and in the other tests) to ensure that the backend has @@ -41,9 +54,7 @@ def test_transformer_zero_elevation_enabled(page: Page, playwright_helper, tmp_p extract_and_upload(page, "Synthetic_Zero_Elevation.gpx.gz", tmp_path) - page.locator("#transformer-heading-1 .accordion-button").click() - page.locator("#transformer-1.collapse.show").wait_for() - page.locator("#transformer-enabled-1").check() + set_transformers(page, [True, False, False, False]) page.locator(".btn", has_text="Upload").click() @@ -62,14 +73,13 @@ def test_transformer_zero_elevation_edited(page: Page, playwright_helper, tmp_pa page.get_by_text("Upload").click() extract_and_upload(page, "Synthetic_Zero_Elevation.gpx.gz", tmp_path) + set_transformers(page, [False, False, False, False]) page.locator(".btn", has_text="Upload").click() page.locator(".btn", has_text="Edit").click() - page.locator("#transformer-heading-1 .accordion-button").click() - page.locator("#transformer-1.collapse.show").wait_for() - page.locator("#transformer-enabled-1").check() + set_transformers(page, [True, False, False, False]) page.locator(".btn", has_text="Save").click() @@ -87,6 +97,7 @@ def test_transformer_steep_slope_disabled(page: Page, playwright_helper, tmp_pat page.get_by_text("Upload").click() extract_and_upload(page, "Synthetic_Steep_Slope.gpx.gz", tmp_path) + set_transformers(page, [False, False, False, False]) page.locator(".btn", has_text="Upload").click() @@ -104,10 +115,7 @@ def test_transformer_steep_slope_enabled(page: Page, playwright_helper, tmp_path page.get_by_text("Upload").click() extract_and_upload(page, "Synthetic_Steep_Slope.gpx.gz", tmp_path) - - page.locator("#transformer-heading-1 .accordion-button").click() - page.locator("#transformer-1.collapse.show").wait_for() - page.locator("#transformer-enabled-1").check() + set_transformers(page, [True, False, False, False]) page.locator(".btn", has_text="Upload").click() @@ -126,14 +134,13 @@ def test_transformer_steep_slope_edited(page: Page, playwright_helper, tmp_path, page.get_by_text("Upload").click() extract_and_upload(page, "Synthetic_Steep_Slope.gpx.gz", tmp_path) + set_transformers(page, [False, False, False, False]) page.locator(".btn", has_text="Upload").click() page.locator(".btn", has_text="Edit").click() - page.locator("#transformer-heading-1 .accordion-button").click() - page.locator("#transformer-1.collapse.show").wait_for() - page.locator("#transformer-enabled-1").check() + set_transformers(page, [True, False, False, False]) page.locator(".btn", has_text="Save").click() @@ -151,10 +158,7 @@ def test_transformer_elevation_jump_enabled(page: Page, playwright_helper, tmp_p page.get_by_text("Upload").click() extract_and_upload(page, "Elevation_Jump.gpx.gz", tmp_path) - - page.locator("#transformer-heading-2 .accordion-button").click() - page.locator("#transformer-2.collapse.show").wait_for() - page.locator("#transformer-enabled-2").check() + set_transformers(page, [False, True, False, False]) page.locator(".btn", has_text="Upload").click() |
