diff options
| -rw-r--r-- | CHANGELOG.rst | 13 | ||||
| -rw-r--r-- | fietsboek/__init__.py | 7 | ||||
| -rw-r--r-- | fietsboek/actions.py | 57 | ||||
| -rw-r--r-- | fietsboek/data.py | 125 | ||||
| -rw-r--r-- | fietsboek/locale/de/LC_MESSAGES/messages.mo | bin | 12059 -> 12157 bytes | |||
| -rw-r--r-- | fietsboek/locale/de/LC_MESSAGES/messages.po | 63 | ||||
| -rw-r--r-- | fietsboek/locale/en/LC_MESSAGES/messages.mo | bin | 11347 -> 11441 bytes | |||
| -rw-r--r-- | fietsboek/locale/en/LC_MESSAGES/messages.po | 59 | ||||
| -rw-r--r-- | fietsboek/locale/fietslog.pot | 59 | ||||
| -rw-r--r-- | fietsboek/models/meta.py | 2 | ||||
| -rw-r--r-- | fietsboek/summaries.py | 9 | ||||
| -rw-r--r-- | fietsboek/templates/home.jinja2 | 22 | ||||
| -rw-r--r-- | fietsboek/templates/layout.jinja2 | 3 | ||||
| -rw-r--r-- | fietsboek/views/edit.py | 4 | ||||
| -rw-r--r-- | fietsboek/views/tileproxy.py | 87 | ||||
| -rw-r--r-- | fietsboek/views/upload.py | 8 | ||||
| -rw-r--r-- | tests/conftest.py | 6 |
17 files changed, 386 insertions, 138 deletions
diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 8b08927..db73b5b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -8,6 +8,19 @@ Added ^^^^^ - The maintenance mode. +- The summary on the home page now shows the number of tracks per time period. +- The summary on the home page now shows the track length at first glance. + +Changed +^^^^^^^ + +- Python 3.9 is the new minimum Python version (up from 3.7). + +Fixed +^^^^^ + +- Page reading for systems that use a non-UTF-8 locale. +- The filename above the map is hidden again. 0.5.0 - 2023-01-12 ------------------ diff --git a/fietsboek/__init__.py b/fietsboek/__init__.py index 95ff394..41e8a04 100644 --- a/fietsboek/__init__.py +++ b/fietsboek/__init__.py @@ -90,7 +90,12 @@ def maintenance_mode( def main(_global_config, **settings): """This function returns a Pyramid WSGI application.""" + # Avoid a circular import by not importing at the top level + # pylint: disable=import-outside-toplevel,cyclic-import + from .views.tileproxy import TileRequester + parsed_config = mod_config.parse(settings) + settings["jinja2.newstyle"] = True def data_manager(request): return DataManager(Path(request.config.data_dir)) @@ -134,6 +139,8 @@ def main(_global_config, **settings): config.add_request_method(redis_, name="redis", reify=True) config.add_request_method(config_, name="config", reify=True) + config.registry.registerUtility(TileRequester()) + jinja2_env = config.get_jinja2_environment() jinja2_env.filters["format_decimal"] = mod_jinja2.filter_format_decimal jinja2_env.filters["format_datetime"] = mod_jinja2.filter_format_datetime diff --git a/fietsboek/actions.py b/fietsboek/actions.py index 16631b2..e49e27d 100644 --- a/fietsboek/actions.py +++ b/fietsboek/actions.py @@ -16,10 +16,10 @@ from pyramid.request import Request from sqlalchemy import select from sqlalchemy.orm.session import Session -from . import models +from . import models, util from . import transformers as mod_transformers from . import util -from .data import DataManager +from .data import DataManager, TrackDataDir from .models.track import TrackType, Visibility LOGGER = logging.getLogger(__name__) @@ -83,35 +83,34 @@ def add_track( # Save the GPX data LOGGER.debug("Creating a new data folder for %d", track.id) - manager = data_manager.initialize(track.id) - LOGGER.debug("Saving GPX to %s", manager.gpx_path()) - manager.compress_gpx(gpx_data) - manager.backup() - - gpx = gpxpy.parse(gpx_data) - for transformer in transformers: - LOGGER.debug("Running %s with %r", transformer, transformer.parameters) - transformer.execute(gpx) - manager.compress_gpx(gpx.to_xml().encode("utf-8")) - track.transformers = [[tfm.identifier(), tfm.parameters.dict()] for tfm in transformers] - - # Best time to build the cache is right after the upload, but *after* the - # transformers have been applied! - track.ensure_cache(gpx) - dbsession.add(track.cache) - - manager.engrave_metadata( - title=track.title, - description=track.description, - author_name=track.owner.name, - time=track.date, - gpx=gpx, - ) + with data_manager.initialize(track.id) as manager: + LOGGER.debug("Saving GPX to %s", manager.gpx_path()) + manager.compress_gpx(gpx_data) + manager.backup() + + gpx = gpxpy.parse(gpx_data) + for transformer in transformers: + LOGGER.debug("Running %s with %r", transformer, transformer.parameters) + transformer.execute(gpx) + track.transformers = [[tfm.identifier(), tfm.parameters.dict()] for tfm in transformers] + + # Best time to build the cache is right after the upload, but *after* the + # transformers have been applied! + track.ensure_cache(gpx) + dbsession.add(track.cache) + + manager.engrave_metadata( + title=track.title, + description=track.description, + author_name=track.owner.name, + time=track.date, + gpx=gpx, + ) return track -def edit_images(request: Request, track: models.Track): +def edit_images(request: Request, track: models.Track, *, manager: Optional[TrackDataDir] = None): """Edit the images according to the given request. This deletes and adds images and image descriptions as needed, based on the @@ -119,9 +118,11 @@ def edit_images(request: Request, track: models.Track): :param request: The request. :param track: The track to edit. + :param manager: The track's data manager. If not given, it will + automatically be opened from the request. """ LOGGER.debug("Editing images for %d", track.id) - manager = request.data_manager.open(track.id) + manager = manager if manager else request.data_manager.open(track.id) # Delete requested images for image in request.params.getall("delete-image[]"): diff --git a/fietsboek/data.py b/fietsboek/data.py index 6906c84..9f49c49 100644 --- a/fietsboek/data.py +++ b/fietsboek/data.py @@ -10,7 +10,7 @@ import shutil import string import uuid from pathlib import Path -from typing import BinaryIO, List, Optional +from typing import BinaryIO, List, Literal, Optional import brotli import gpxpy @@ -77,7 +77,7 @@ class DataManager: """ path = self._track_data_dir(track_id) path.mkdir(parents=True) - return TrackDataDir(track_id, path) + return TrackDataDir(track_id, path, journal=True, is_fresh=True) def purge(self, track_id: int): """Forcefully purges all data from the given track. @@ -101,11 +101,94 @@ class DataManager: class TrackDataDir: - """Manager for a single track's data.""" + """Manager for a single track's data. + + If initialized with ``journal = True``, then you can use :meth:`rollback` + to roll back the changes in case of an error. In case of no error, use + :meth:`commit` to commit the changes. If you don't want the "journalling" + semantics, use ``journal = False``. + """ - def __init__(self, track_id: int, path: Path): + def __init__(self, track_id: int, path: Path, *, journal: bool = False, is_fresh: bool = False): self.track_id: int = track_id self.path: Path = path + self.journal: Optional[list] = [] if journal else None + self.is_fresh = is_fresh + + def __enter__(self) -> "TrackDataDir": + if self.journal is None: + self.journal = [] + return self + + def __exit__(self, exc_type, exc_val, exc_tb) -> Literal[False]: + if exc_type is None and exc_val is None and exc_tb is None: + self.commit() + else: + self.rollback() + return False + + def rollback(self): + """Rolls back the journal, e.g. in case of error. + + :raises ValueError: If the data directory was opened without the + journal, this raises :exc:`ValueError`. + """ + LOGGER.debug("Rolling back state of %s", self.path) + + if self.journal is None: + raise ValueError("Rollback on a non-journalling data directory") + + if self.is_fresh: + # Shortcut if the directory is fresh, simply remove everything + self.journal = None + self.purge() + return + + for action, *rest in reversed(self.journal): + if action == "purge": + (new_name,) = rest + shutil.move(new_name, self.path) + elif action == "compress_gpx": + (old_data,) = rest + if old_data is None: + self.gpx_path().unlink() + else: + self.gpx_path().write_bytes(old_data) + elif action == "add_image": + (image_path,) = rest + image_path.unlink() + elif action == "delete_image": + path, data = rest + path.write_bytes(data) + + self.journal = None + + def commit(self): + """Commits all changed and deletes the journal. + + Note that this function will do nothing if the journal is disabled, + meaning it can always be called. + """ + LOGGER.debug("Committing journal for %s", self.path) + + if self.journal is None: + return + + for action, *rest in reversed(self.journal): + if action == "purge": + (new_name,) = rest + shutil.rmtree(new_name, ignore_errors=False, onerror=self._log_deletion_error) + elif action == "compress_gpx": + # Nothing to do here, the new data is already on the disk + pass + elif action == "add_image": + # Nothing to do here, the image is already saved + pass + elif action == "delete_image": + # Again, nothing to do here, we simply discard the in-memory image data + pass + + self.journal = None def lock(self) -> FileLock: """Returns a FileLock that can be used to lock access to the track's @@ -115,18 +198,23 @@ class TrackDataDir: """ return FileLock(self.path / "lock") + @staticmethod + def _log_deletion_error(_, path, exc_info): + LOGGER.warning("Failed to remove %s", path, exc_info=exc_info) + def purge(self): """Purge all data pertaining to the track. This function logs errors but raises no exception, as such it can always be used to clean up after a track. """ - - def log_error(_, path, exc_info): - LOGGER.warning("Failed to remove %s", path, exc_info=exc_info) - - if self.path.is_dir(): - shutil.rmtree(self.path, ignore_errors=False, onerror=log_error) + if self.journal is None: + if self.path.is_dir(): + shutil.rmtree(self.path, ignore_errors=False, onerror=self._log_deletion_error) + else: + new_name = self.path.with_name("trash-" + self.path.name) + shutil.move(self.path, new_name) + self.journal.append(("purge", new_name)) def gpx_path(self) -> Path: """Returns the path of the GPX file. @@ -147,6 +235,16 @@ class TrackDataDir: :param quality: Compression quality, from 0 to 11 - 11 is highest quality but slowest compression speed. """ + if self.journal is not None: + # First, we check if we already saved an old state of the GPX data + for action, *_ in self.journal: + if action == "compress_gpx": + break + else: + # We did not save a state yet + old_data = None if not self.gpx_path().is_file() else self.gpx_path().read_bytes() + self.journal.append(("compress_gpx", old_data)) + compressed = brotli.compress(data, quality=quality) self.gpx_path().write_bytes(compressed) @@ -241,6 +339,9 @@ class TrackDataDir: with open(path, "wb") as fobj: shutil.copyfileobj(image, fobj) + if self.journal is not None: + self.journal.append(("add_image", path)) + return filename def delete_image(self, image_id: str): @@ -254,4 +355,8 @@ class TrackDataDir: if "/" in image_id or "\\" in image_id: return path = self.image_path(image_id) + + if self.journal is not None: + self.journal.append(("delete_image", path, path.read_bytes())) + path.unlink() diff --git a/fietsboek/locale/de/LC_MESSAGES/messages.mo b/fietsboek/locale/de/LC_MESSAGES/messages.mo Binary files differindex 6043d8e..a79349d 100644 --- a/fietsboek/locale/de/LC_MESSAGES/messages.mo +++ b/fietsboek/locale/de/LC_MESSAGES/messages.mo diff --git a/fietsboek/locale/de/LC_MESSAGES/messages.po b/fietsboek/locale/de/LC_MESSAGES/messages.po index 8787ec2..8b59ecb 100644 --- a/fietsboek/locale/de/LC_MESSAGES/messages.po +++ b/fietsboek/locale/de/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2023-02-15 23:01+0100\n" +"POT-Creation-Date: 2023-03-07 20:11+0100\n" "PO-Revision-Date: 2022-07-02 17:35+0200\n" "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" "Language: de\n" @@ -18,39 +18,39 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 2.11.0\n" -#: fietsboek/util.py:273 +#: fietsboek/util.py:276 msgid "password_constraint.mismatch" msgstr "Passwörter stimmen nicht überein" -#: fietsboek/util.py:275 +#: fietsboek/util.py:278 msgid "password_constraint.length" msgstr "Passwort zu kurz" -#: fietsboek/models/track.py:565 +#: fietsboek/models/track.py:566 msgid "tooltip.table.length" msgstr "Länge" -#: fietsboek/models/track.py:566 +#: fietsboek/models/track.py:567 msgid "tooltip.table.uphill" msgstr "Bergauf" -#: fietsboek/models/track.py:567 +#: fietsboek/models/track.py:568 msgid "tooltip.table.downhill" msgstr "Bergab" -#: fietsboek/models/track.py:568 +#: fietsboek/models/track.py:569 msgid "tooltip.table.moving_time" msgstr "Fahrzeit" -#: fietsboek/models/track.py:569 +#: fietsboek/models/track.py:570 msgid "tooltip.table.stopped_time" msgstr "Haltezeit" -#: fietsboek/models/track.py:571 +#: fietsboek/models/track.py:572 msgid "tooltip.table.max_speed" msgstr "Maximalgeschwindigkeit" -#: fietsboek/models/track.py:575 +#: fietsboek/models/track.py:576 msgid "tooltip.table.avg_speed" msgstr "Durchschnittsgeschwindigkeit" @@ -460,51 +460,58 @@ msgstr "Abbrechen" msgid "page.home.title" msgstr "Startseite" -#: fietsboek/templates/home.jinja2:23 +#: fietsboek/templates/home.jinja2:12 fietsboek/templates/home.jinja2:19 +#: fietsboek/templates/home.jinja2:37 +msgid "page.home.summary.track" +msgid_plural "page.home.summary.tracks" +msgstr[0] "%(num)d Strecke" +msgstr[1] "%(num)d Strecken" + +#: fietsboek/templates/home.jinja2:37 msgid "page.home.total" msgstr "Gesamt" -#: fietsboek/templates/layout.jinja2:36 +#: fietsboek/templates/layout.jinja2:39 msgid "page.navbar.toggle" msgstr "Navigation umschalten" -#: fietsboek/templates/layout.jinja2:47 +#: fietsboek/templates/layout.jinja2:50 msgid "page.navbar.home" msgstr "Startseite" -#: fietsboek/templates/layout.jinja2:50 +#: fietsboek/templates/layout.jinja2:53 msgid "page.navbar.browse" msgstr "Stöbern" -#: fietsboek/templates/layout.jinja2:54 +#: fietsboek/templates/layout.jinja2:57 msgid "page.navbar.upload" msgstr "Hochladen" -#: fietsboek/templates/layout.jinja2:63 +#: fietsboek/templates/layout.jinja2:66 msgid "page.navbar.user" msgstr "Nutzer" -#: fietsboek/templates/layout.jinja2:67 +#: fietsboek/templates/layout.jinja2:70 msgid "page.navbar.welcome_user" msgstr "Willkommen, {}!" -#: fietsboek/templates/layout.jinja2:70 +#: fietsboek/templates/layout.jinja2:73 msgid "page.navbar.logout" msgstr "Abmelden" -#: fietsboek/templates/layout.jinja2:73 +#: fietsboek/templates/layout.jinja2:76 msgid "page.navbar.profile" msgstr "Profil" -#: fietsboek/templates/layout.jinja2:77 +#: fietsboek/templates/layout.jinja2:80 msgid "page.navbar.admin" msgstr "Admin" -#: fietsboek/templates/layout.jinja2:83 +#: fietsboek/templates/layout.jinja2:86 msgid "page.navbar.login" msgstr "Anmelden" -#: fietsboek/templates/layout.jinja2:87 +#: fietsboek/templates/layout.jinja2:90 msgid "page.navbar.create_account" msgstr "Konto Erstellen" @@ -630,13 +637,15 @@ msgstr "Anfrage senden" msgid "page.upload.form.gpx" msgstr "GPX Datei" -#: fietsboek/transformers/__init__.py:130 +#: fietsboek/transformers/__init__.py:140 msgid "transformers.fix-null-elevation.title" msgstr "Nullhöhen beheben" -#: fietsboek/transformers/__init__.py:134 +#: fietsboek/transformers/__init__.py:144 msgid "transformers.fix-null-elevation.description" -msgstr "Diese Transformation passt die Höhenangabe für Punkte an, bei denen die Höhenangabe fehlt." +msgstr "" +"Diese Transformation passt die Höhenangabe für Punkte an, bei denen die " +"Höhenangabe fehlt." #: fietsboek/views/account.py:54 msgid "flash.invalid_name" @@ -749,11 +758,11 @@ msgstr "Keine Datei ausgewählt" msgid "flash.invalid_file" msgstr "Ungültige GPX-Datei gesendet" -#: fietsboek/views/upload.py:182 +#: fietsboek/views/upload.py:188 msgid "flash.upload_success" msgstr "Hochladen erfolgreich" -#: fietsboek/views/upload.py:198 +#: fietsboek/views/upload.py:204 msgid "flash.upload_cancelled" msgstr "Hochladen abgebrochen" diff --git a/fietsboek/locale/en/LC_MESSAGES/messages.mo b/fietsboek/locale/en/LC_MESSAGES/messages.mo Binary files differindex 4f35491..13debb5 100644 --- a/fietsboek/locale/en/LC_MESSAGES/messages.mo +++ b/fietsboek/locale/en/LC_MESSAGES/messages.mo diff --git a/fietsboek/locale/en/LC_MESSAGES/messages.po b/fietsboek/locale/en/LC_MESSAGES/messages.po index 5f3ab68..632e0b7 100644 --- a/fietsboek/locale/en/LC_MESSAGES/messages.po +++ b/fietsboek/locale/en/LC_MESSAGES/messages.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2023-02-15 23:01+0100\n" +"POT-Creation-Date: 2023-03-07 20:11+0100\n" "PO-Revision-Date: 2022-06-28 13:11+0200\n" "Last-Translator: \n" "Language: en\n" @@ -18,39 +18,39 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 2.11.0\n" -#: fietsboek/util.py:273 +#: fietsboek/util.py:276 msgid "password_constraint.mismatch" msgstr "Passwords don't match" -#: fietsboek/util.py:275 +#: fietsboek/util.py:278 msgid "password_constraint.length" msgstr "Password not long enough" -#: fietsboek/models/track.py:565 +#: fietsboek/models/track.py:566 msgid "tooltip.table.length" msgstr "Length" -#: fietsboek/models/track.py:566 +#: fietsboek/models/track.py:567 msgid "tooltip.table.uphill" msgstr "Uphill" -#: fietsboek/models/track.py:567 +#: fietsboek/models/track.py:568 msgid "tooltip.table.downhill" msgstr "Downhill" -#: fietsboek/models/track.py:568 +#: fietsboek/models/track.py:569 msgid "tooltip.table.moving_time" msgstr "Moving Time" -#: fietsboek/models/track.py:569 +#: fietsboek/models/track.py:570 msgid "tooltip.table.stopped_time" msgstr "Stopped Time" -#: fietsboek/models/track.py:571 +#: fietsboek/models/track.py:572 msgid "tooltip.table.max_speed" msgstr "Max Speed" -#: fietsboek/models/track.py:575 +#: fietsboek/models/track.py:576 msgid "tooltip.table.avg_speed" msgstr "Average Speed" @@ -456,51 +456,58 @@ msgstr "Cancel" msgid "page.home.title" msgstr "Home" -#: fietsboek/templates/home.jinja2:23 +#: fietsboek/templates/home.jinja2:12 fietsboek/templates/home.jinja2:19 +#: fietsboek/templates/home.jinja2:37 +msgid "page.home.summary.track" +msgid_plural "page.home.summary.tracks" +msgstr[0] "%(num)d track" +msgstr[1] "%(num)d tracks" + +#: fietsboek/templates/home.jinja2:37 msgid "page.home.total" msgstr "Total" -#: fietsboek/templates/layout.jinja2:36 +#: fietsboek/templates/layout.jinja2:39 msgid "page.navbar.toggle" msgstr "Toggle navigation" -#: fietsboek/templates/layout.jinja2:47 +#: fietsboek/templates/layout.jinja2:50 msgid "page.navbar.home" msgstr "Home" -#: fietsboek/templates/layout.jinja2:50 +#: fietsboek/templates/layout.jinja2:53 msgid "page.navbar.browse" msgstr "Browse" -#: fietsboek/templates/layout.jinja2:54 +#: fietsboek/templates/layout.jinja2:57 msgid "page.navbar.upload" msgstr "Upload" -#: fietsboek/templates/layout.jinja2:63 +#: fietsboek/templates/layout.jinja2:66 msgid "page.navbar.user" msgstr "User" -#: fietsboek/templates/layout.jinja2:67 +#: fietsboek/templates/layout.jinja2:70 msgid "page.navbar.welcome_user" msgstr "Welcome, {}!" -#: fietsboek/templates/layout.jinja2:70 +#: fietsboek/templates/layout.jinja2:73 msgid "page.navbar.logout" msgstr "Logout" -#: fietsboek/templates/layout.jinja2:73 +#: fietsboek/templates/layout.jinja2:76 msgid "page.navbar.profile" msgstr "Profile" -#: fietsboek/templates/layout.jinja2:77 +#: fietsboek/templates/layout.jinja2:80 msgid "page.navbar.admin" msgstr "Admin" -#: fietsboek/templates/layout.jinja2:83 +#: fietsboek/templates/layout.jinja2:86 msgid "page.navbar.login" msgstr "Login" -#: fietsboek/templates/layout.jinja2:87 +#: fietsboek/templates/layout.jinja2:90 msgid "page.navbar.create_account" msgstr "Create Account" @@ -626,11 +633,11 @@ msgstr "Send request" msgid "page.upload.form.gpx" msgstr "GPX file" -#: fietsboek/transformers/__init__.py:130 +#: fietsboek/transformers/__init__.py:140 msgid "transformers.fix-null-elevation.title" msgstr "Fix null elevation" -#: fietsboek/transformers/__init__.py:134 +#: fietsboek/transformers/__init__.py:144 msgid "transformers.fix-null-elevation.description" msgstr "This transformer fixes the elevation of points whose elevation is unset." @@ -744,11 +751,11 @@ msgstr "No file selected" msgid "flash.invalid_file" msgstr "Invalid GPX file selected" -#: fietsboek/views/upload.py:182 +#: fietsboek/views/upload.py:188 msgid "flash.upload_success" msgstr "Upload successful" -#: fietsboek/views/upload.py:198 +#: fietsboek/views/upload.py:204 msgid "flash.upload_cancelled" msgstr "Upload cancelled" diff --git a/fietsboek/locale/fietslog.pot b/fietsboek/locale/fietslog.pot index 05bc70a..70f010a 100644 --- a/fietsboek/locale/fietslog.pot +++ b/fietsboek/locale/fietslog.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PROJECT VERSION\n" "Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2023-02-15 23:01+0100\n" +"POT-Creation-Date: 2023-03-07 20:11+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n" "Language-Team: LANGUAGE <LL@li.org>\n" @@ -17,39 +17,39 @@ msgstr "" "Content-Transfer-Encoding: 8bit\n" "Generated-By: Babel 2.11.0\n" -#: fietsboek/util.py:273 +#: fietsboek/util.py:276 msgid "password_constraint.mismatch" msgstr "" -#: fietsboek/util.py:275 +#: fietsboek/util.py:278 msgid "password_constraint.length" msgstr "" -#: fietsboek/models/track.py:565 +#: fietsboek/models/track.py:566 msgid "tooltip.table.length" msgstr "" -#: fietsboek/models/track.py:566 +#: fietsboek/models/track.py:567 msgid "tooltip.table.uphill" msgstr "" -#: fietsboek/models/track.py:567 +#: fietsboek/models/track.py:568 msgid "tooltip.table.downhill" msgstr "" -#: fietsboek/models/track.py:568 +#: fietsboek/models/track.py:569 msgid "tooltip.table.moving_time" msgstr "" -#: fietsboek/models/track.py:569 +#: fietsboek/models/track.py:570 msgid "tooltip.table.stopped_time" msgstr "" -#: fietsboek/models/track.py:571 +#: fietsboek/models/track.py:572 msgid "tooltip.table.max_speed" msgstr "" -#: fietsboek/models/track.py:575 +#: fietsboek/models/track.py:576 msgid "tooltip.table.avg_speed" msgstr "" @@ -453,51 +453,58 @@ msgstr "" msgid "page.home.title" msgstr "" -#: fietsboek/templates/home.jinja2:23 +#: fietsboek/templates/home.jinja2:12 fietsboek/templates/home.jinja2:19 +#: fietsboek/templates/home.jinja2:37 +msgid "page.home.summary.track" +msgid_plural "page.home.summary.tracks" +msgstr[0] "" +msgstr[1] "" + +#: fietsboek/templates/home.jinja2:37 msgid "page.home.total" msgstr "" -#: fietsboek/templates/layout.jinja2:36 +#: fietsboek/templates/layout.jinja2:39 msgid "page.navbar.toggle" msgstr "" -#: fietsboek/templates/layout.jinja2:47 +#: fietsboek/templates/layout.jinja2:50 msgid "page.navbar.home" msgstr "" -#: fietsboek/templates/layout.jinja2:50 +#: fietsboek/templates/layout.jinja2:53 msgid "page.navbar.browse" msgstr "" -#: fietsboek/templates/layout.jinja2:54 +#: fietsboek/templates/layout.jinja2:57 msgid "page.navbar.upload" msgstr "" -#: fietsboek/templates/layout.jinja2:63 +#: fietsboek/templates/layout.jinja2:66 msgid "page.navbar.user" msgstr "" -#: fietsboek/templates/layout.jinja2:67 +#: fietsboek/templates/layout.jinja2:70 msgid "page.navbar.welcome_user" msgstr "" -#: fietsboek/templates/layout.jinja2:70 +#: fietsboek/templates/layout.jinja2:73 msgid "page.navbar.logout" msgstr "" -#: fietsboek/templates/layout.jinja2:73 +#: fietsboek/templates/layout.jinja2:76 msgid "page.navbar.profile" msgstr "" -#: fietsboek/templates/layout.jinja2:77 +#: fietsboek/templates/layout.jinja2:80 msgid "page.navbar.admin" msgstr "" -#: fietsboek/templates/layout.jinja2:83 +#: fietsboek/templates/layout.jinja2:86 msgid "page.navbar.login" msgstr "" -#: fietsboek/templates/layout.jinja2:87 +#: fietsboek/templates/layout.jinja2:90 msgid "page.navbar.create_account" msgstr "" @@ -621,11 +628,11 @@ msgstr "" msgid "page.upload.form.gpx" msgstr "" -#: fietsboek/transformers/__init__.py:130 +#: fietsboek/transformers/__init__.py:140 msgid "transformers.fix-null-elevation.title" msgstr "" -#: fietsboek/transformers/__init__.py:134 +#: fietsboek/transformers/__init__.py:144 msgid "transformers.fix-null-elevation.description" msgstr "" @@ -733,11 +740,11 @@ msgstr "" msgid "flash.invalid_file" msgstr "" -#: fietsboek/views/upload.py:182 +#: fietsboek/views/upload.py:188 msgid "flash.upload_success" msgstr "" -#: fietsboek/views/upload.py:198 +#: fietsboek/views/upload.py:204 msgid "flash.upload_cancelled" msgstr "" diff --git a/fietsboek/models/meta.py b/fietsboek/models/meta.py index 6b11a09..f06c863 100644 --- a/fietsboek/models/meta.py +++ b/fietsboek/models/meta.py @@ -1,5 +1,5 @@ """Base metadata definition for the SQLAlchemy models.""" -from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import declarative_base from sqlalchemy.schema import MetaData # Recommended naming convention used by Alembic, as various different database diff --git a/fietsboek/summaries.py b/fietsboek/summaries.py index 670c8f2..ed4b197 100644 --- a/fietsboek/summaries.py +++ b/fietsboek/summaries.py @@ -19,6 +19,9 @@ class Summary: items.sort(key=lambda y: y.year) return iter(items) + def __len__(self) -> int: + return len(self.all_tracks()) + def all_tracks(self) -> List[TrackWithMetadata]: """Returns all tracks of the summary. @@ -62,6 +65,9 @@ class YearSummary: items.sort(key=lambda x: x.month) return iter(items) + def __len__(self) -> int: + return len(self.all_tracks()) + def all_tracks(self) -> List[TrackWithMetadata]: """Returns all tracks of the summary. @@ -104,6 +110,9 @@ class MonthSummary: items.sort(key=lambda t: t.date) return iter(items) + def __len__(self) -> int: + return len(self.all_tracks()) + def all_tracks(self) -> List[TrackWithMetadata]: """Returns all tracks of the summary. diff --git a/fietsboek/templates/home.jinja2 b/fietsboek/templates/home.jinja2 index 4a9c3ad..5e552e4 100644 --- a/fietsboek/templates/home.jinja2 +++ b/fietsboek/templates/home.jinja2 @@ -6,13 +6,27 @@ {% if summary %} <div class="list-group list-group-root"> {% for year in summary %} - <a class="list-group-item list-group-item-primary"><i class="bi bi-chevron-down summary-toggler"></i> {{ year.year }} — {{ (year.total_length / 1000) | round(2) | format_decimal }} km</a> + <a class="list-group-item list-group-item-primary"> + <i class="bi bi-chevron-down summary-toggler"></i> + {{ year.year }} + <span class="float-end">{{ ngettext("page.home.summary.track", "page.home.summary.tracks", year|length) }} — {{ (year.total_length / 1000) | round(2) | format_decimal }} km</span> + </a> <div class="list-group collapse show"> {% for month in year %} - <a class="list-group-item list-group-item-secondary"><i class="bi bi-chevron-down summary-toggler"></i> {{ month_name(request, month.month) }} — {{ (month.total_length / 1000) | round(2) | format_decimal }} km</a> + <a class="list-group-item list-group-item-secondary"> + <i class="bi bi-chevron-down summary-toggler"></i> + {{ month_name(request, month.month) }} + <span class="float-end">{{ ngettext("page.home.summary.track", "page.home.summary.tracks", month|length) }} — {{ (month.total_length / 1000) | round(2) | format_decimal }} km</span> + </a> <div class="list-group collapse show"> {% for track in month %} - <span class="list-group-item">{{ track.date.day }}: <a href="{{ request.route_url('details', track_id=track.id) }}" data-bs-toggle="tooltip" data-bs-container="body" data-bs-html="true" title="{{ track.html_tooltip(request.localizer) }}">{{ track.title | default(track.date, true) }}</a></span> + <span class="list-group-item"> + {{ track.date.day }}: + <a href="{{ request.route_url('details', track_id=track.id) }}" data-bs-toggle="tooltip" data-bs-container="body" data-bs-html="true" title="{{ track.html_tooltip(request.localizer) }}"> + {{ track.title | default(track.date, true) }} + </a> + <span class="float-end">{{ (track.length / 1000) | round(2) | format_decimal }} km</span> + </span> {% endfor %} </div> {% endfor %} @@ -20,7 +34,7 @@ {% endfor %} </div> <p> - {{ _("page.home.total") }}: {{ (summary.total_length / 1000) | round(2) | format_decimal }} km + {{ _("page.home.total") }}: {{ ngettext("page.home.summary.track", "page.home.summary.tracks", summary|length) }}, {{ (summary.total_length / 1000) | round(2) | format_decimal }} km </p> {% elif home_content %} {{ home_content | safe }} diff --git a/fietsboek/templates/layout.jinja2 b/fietsboek/templates/layout.jinja2 index 5a10e66..d4e0a0d 100644 --- a/fietsboek/templates/layout.jinja2 +++ b/fietsboek/templates/layout.jinja2 @@ -23,8 +23,11 @@ const FRIENDS_URL = {{ request.route_url('json-friends') | tojson }}; const TILE_LAYERS = {{ embed_tile_layers(request) }}; const BASE_URL = {{ request.route_url('home') | tojson }}; const LOCALE = {{ request.localizer.locale_name.replace('_', '-') | tojson }}; + +// Settings for GPX Viewer. Check GPX2GM_Defs.js for a full list. const Bestaetigung = false; const Fullscreenbutton = true; +const Legende = false; </script> </head> diff --git a/fietsboek/views/edit.py b/fietsboek/views/edit.py index 9cc666f..881f404 100644 --- a/fietsboek/views/edit.py +++ b/fietsboek/views/edit.py @@ -82,7 +82,7 @@ def do_edit(request): data: TrackDataDir = request.data_manager.open(track.id) tz_offset = datetime.timedelta(minutes=int(request.params["date-tz"])) date = datetime.datetime.fromisoformat(request.params["date"]) - with data.lock(): + with data, data.lock(): track.date = date.replace(tzinfo=datetime.timezone(tz_offset)) track.tagged_people = tagged_people @@ -94,7 +94,7 @@ def do_edit(request): tags = request.params.getall("tag[]") track.sync_tags(tags) - actions.edit_images(request, request.context) + actions.edit_images(request, request.context, manager=data) gpx = actions.execute_transformers(request, request.context) data.engrave_metadata( title=track.title, diff --git a/fietsboek/views/tileproxy.py b/fietsboek/views/tileproxy.py index 87a742d..5ed79a5 100644 --- a/fietsboek/views/tileproxy.py +++ b/fietsboek/views/tileproxy.py @@ -9,8 +9,9 @@ Additionally, this protects the users' IP, as only fietsboek can see it. import datetime import logging import random +import threading from itertools import chain -from typing import List +from typing import List, Optional import requests from pyramid.httpexceptions import HTTPBadRequest, HTTPGatewayTimeout @@ -18,6 +19,7 @@ from pyramid.request import Request from pyramid.response import Response from pyramid.view import view_config from requests.exceptions import ReadTimeout +from zope.interface import Interface, implementer from .. import __VERSION__ from ..config import Config, LayerAccess, LayerType, TileLayerConfig @@ -37,7 +39,7 @@ DEFAULT_TILE_LAYERS = [ TileLayerConfig( layer_id="osm", name="OSM", - url="https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", + url="https://tile.openstreetmap.org/{z}/{x}/{y}.png", layer_type=LayerType.BASE, zoom=19, access=LayerAccess.PUBLIC, @@ -72,7 +74,7 @@ DEFAULT_TILE_LAYERS = [ TileLayerConfig( layer_id="osmde", name="OSMDE", - url="https://{s}.tile.openstreetmap.de/tiles/osmde/{z}/{x}/{y}.png", + url="https://tile.openstreetmap.de/tiles/osmde/{z}/{x}/{y}.png", layer_type=LayerType.BASE, zoom=19, access=LayerAccess.PUBLIC, @@ -89,7 +91,7 @@ DEFAULT_TILE_LAYERS = [ TileLayerConfig( layer_id="opentopo", name="Open Topo", - url="https://{s}.tile.opentopomap.org/{z}/{x}/{y}.png", + url="https://tile.opentopomap.org/{z}/{x}/{y}.png", layer_type=LayerType.BASE, zoom=17, access=LayerAccess.PUBLIC, @@ -217,6 +219,71 @@ PUNISHMENT_TTL = datetime.timedelta(minutes=10) PUNISHMENT_THRESHOLD = 10 """Block a provider after that many requests have timed out.""" +MAX_CONCURRENT_CONNECTIONS = 2 +"""Maximum TCP connections per tile host.""" + +CONNECTION_CLOSE_TIMEOUT = datetime.timedelta(seconds=2) +"""Timeout after which keep-alive connections are killed. + +Note that new requests reset the timeout. +""" + + +class ITileRequester(Interface): # pylint: disable=inherit-non-class + """An interface to define the tile requester.""" + + def load_tile(self, url: str, headers: Optional[dict[str, str]] = None) -> requests.Response: + """Loads a tile at the given URL. + + :param url: The URL of the tile to load. + :param headers: Additional headers to send. + :return: The response. + """ + raise NotImplementedError() + + +@implementer(ITileRequester) +class TileRequester: # pylint: disable=too-few-public-methods + """Implementation of the tile requester using requests sessions. + + The benefit of this over doing ``requests.get`` is that we can re-use + connections, and we ensure that we comply with the use policy of the tile + servers by not hammering them with too many connections. + """ + + def __init__(self): + self.session = requests.Session() + adapter = requests.adapters.HTTPAdapter( + pool_maxsize=MAX_CONCURRENT_CONNECTIONS, + pool_block=True, + ) + self.session.mount("http://", adapter) + self.session.mount("https://", adapter) + self.lock = threading.Lock() + self.closer = None + + def load_tile(self, url: str, headers: Optional[dict[str, str]] = None) -> requests.Response: + """Implementation of :meth:`ITileRequester.load_tile`.""" + response = self.session.get(url, headers=headers, timeout=TIMEOUT.total_seconds()) + response.raise_for_status() + self._schedule_session_close() + return response + + def _schedule_session_close(self): + with self.lock: + if self.closer: + self.closer.cancel() + self.closer = threading.Timer( + CONNECTION_CLOSE_TIMEOUT.total_seconds(), + self._close_session, + ) + self.closer.start() + + def _close_session(self): + with self.lock: + self.closer = None + self.session.close() + @view_config(route_name="tile-proxy", http_cache=3600) def tile_proxy(request): @@ -227,6 +294,7 @@ def tile_proxy(request): :return: The HTTP response. :rtype: pyramid.response.Response """ + # pylint: disable=too-many-locals if request.config.disable_tile_proxy: raise HTTPBadRequest("Tile proxying is disabled") @@ -262,19 +330,18 @@ def tile_proxy(request): if from_mail: headers["from"] = from_mail + loader: ITileRequester = request.registry.getUtility(ITileRequester) try: - resp = requests.get(url, headers=headers, timeout=TIMEOUT.total_seconds()) + resp = loader.load_tile(url, headers=headers) except ReadTimeout: LOGGER.debug("Proxy timeout when accessing %r", url) request.redis.incr(timeout_tracker) request.redis.expire(timeout_tracker, PUNISHMENT_TTL) raise HTTPGatewayTimeout(f"No response in time from {provider}") from None + except requests.HTTPError as exc: + LOGGER.info("Proxy request failed for %s: %s", provider, exc) + return Response(f"Failed to get tile from {provider}", status_code=exc.response.status_code) else: - try: - resp.raise_for_status() - except requests.HTTPError as exc: - LOGGER.info("Proxy request failed for %s: %s", provider, exc) - return Response(f"Failed to get tile from {provider}", status_code=resp.status_code) request.redis.set(cache_key, resp.content, ex=TTL) return Response(resp.content, content_type=resp.headers.get("Content-type", content_type)) diff --git a/fietsboek/views/upload.py b/fietsboek/views/upload.py index 15fa990..fd93034 100644 --- a/fietsboek/views/upload.py +++ b/fietsboek/views/upload.py @@ -177,7 +177,13 @@ def do_finish_upload(request): # Don't forget to add the images LOGGER.debug("Starting to edit images for the upload") - actions.edit_images(request, track) + try: + actions.edit_images(request, track) + except Exception: + # We just created the folder, so we'll be fine deleting it + LOGGER.info("Deleting partially created folder for track %d", track.id) + request.data_manager.open(track.id).purge() + raise request.session.flash(request.localizer.translate(_("flash.upload_success"))) diff --git a/tests/conftest.py b/tests/conftest.py index a203775..a499bec 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -59,9 +59,9 @@ def data_manager(app_settings): def _cleanup_data(app_settings): yield engine = models.get_engine(app_settings) - connection = engine.connect() - for table in reversed(Base.metadata.sorted_tables): - connection.execute(table.delete()) + with engine.begin() as connection: + for table in reversed(Base.metadata.sorted_tables): + connection.execute(table.delete()) data_dir = Path(app_settings["fietsboek.data_dir"]) if (data_dir / "tracks").is_dir(): shutil.rmtree(data_dir / "tracks") |