diff options
| -rw-r--r-- | fietsboek/__init__.py | 2 | ||||
| -rw-r--r-- | fietsboek/trackmap.py | 14 | ||||
| -rw-r--r-- | fietsboek/views/tileproxy.py | 98 |
3 files changed, 79 insertions, 35 deletions
diff --git a/fietsboek/__init__.py b/fietsboek/__init__.py index 3d6c125..797d38b 100644 --- a/fietsboek/__init__.py +++ b/fietsboek/__init__.py @@ -174,7 +174,7 @@ def main(global_config, **settings): config.add_request_method(redis_, name="redis", reify=True) config.add_request_method(config_, name="config", reify=True) - config.registry.registerUtility(TileRequester()) + config.registry.registerUtility(TileRequester(redis.from_url(parsed_config.redis_url))) jinja2_env = config.get_jinja2_environment() jinja2_env.filters["format_decimal"] = mod_jinja2.filter_format_decimal diff --git a/fietsboek/trackmap.py b/fietsboek/trackmap.py index b141790..a9fc439 100644 --- a/fietsboek/trackmap.py +++ b/fietsboek/trackmap.py @@ -79,18 +79,8 @@ class TrackMapRenderer: return start_x, start_y def _load_tile(self, zoom, x, y) -> Image: - url = ( - self.layer.url - .unicode_string() - .replace(quote("{x}"), str(x)) - .replace(quote("{y}"), str(y)) - .replace(quote("{z}"), str(zoom)) - ) - headers = { - "user-agent": f"Fietsboek-Tile-Proxy/0.10.0", - } - imagebytes = self.requester.load_tile(url, headers=headers).content - return Image.open(io.BytesIO(imagebytes)) + tile_data = self.requester.load_tile(self.layer, zoom, x, y) + return Image.open(io.BytesIO(tile_data)) def _draw_lines(self, image, zoom, start_x, start_y): starts = self.track.walk(only_points=True) diff --git a/fietsboek/views/tileproxy.py b/fietsboek/views/tileproxy.py index 0efe4de..11abf2a 100644 --- a/fietsboek/views/tileproxy.py +++ b/fietsboek/views/tileproxy.py @@ -242,15 +242,39 @@ Note that new requests reset the timeout. class ITileRequester(Interface): # pylint: disable=inherit-non-class """An interface to define the tile requester.""" - def load_tile(self, url: str, headers: Optional[dict[str, str]] = None) -> requests.Response: + def load_url(self, url: str, headers: Optional[dict[str, str]] = None) -> requests.Response: """Loads a tile at the given URL. + This is a low-level version of :meth:`~ITileRequester.load_tile`. + :param url: The URL of the tile to load. :param headers: Additional headers to send. :return: The response. """ raise NotImplementedError() + def load_tile( + self, + layer: TileLayerConfig, + zoom: int, + x: int, + y: int, + *, + headers: Optional[dict[str, str]] = None, + use_cache: bool = True, + ) -> bytes: + """Loads a tile from the given layer. + + :param layer: The configured tile layer. + :param zoom: The zoom level. + :param x: The tile's x coordinate. + :param y: The tile's y coordinate. + :param headers: Additional headers. + :param use_cache: Whether to use the cache (if available). + :return: The bytes of the tile. + """ + raise NotImplementedError() + @implementer(ITileRequester) class TileRequester: # pylint: disable=too-few-public-methods @@ -261,7 +285,7 @@ class TileRequester: # pylint: disable=too-few-public-methods servers by not hammering them with too many connections. """ - def __init__(self): + def __init__(self, redis): self.session = requests.Session() adapter = requests.adapters.HTTPAdapter( pool_maxsize=MAX_CONCURRENT_CONNECTIONS, @@ -271,14 +295,56 @@ class TileRequester: # pylint: disable=too-few-public-methods self.session.mount("https://", adapter) self.lock = threading.Lock() self.closer = None - - def load_tile(self, url: str, headers: Optional[dict[str, str]] = None) -> requests.Response: - """Implementation of :meth:`ITileRequester.load_tile`.""" + self.redis = redis + + def load_url(self, url: str, headers: Optional[dict[str, str]] = None) -> requests.Response: + """Implementation of :meth:`ITileRequester.load_url`.""" + if headers is None: + headers = {} + if "user-agent" not in headers: + headers["user-agent"] = f"Fietsboek/{__VERSION__}" response = self.session.get(url, headers=headers, timeout=TIMEOUT.total_seconds()) response.raise_for_status() self._schedule_session_close() return response + def load_tile( + self, + layer: TileLayerConfig, + zoom: int, + x: int, + y: int, + *, + headers: Optional[dict[str, str]] = None, + use_cache: bool = True, + ) -> bytes: + """Implementation of :meth:`ITileRequester.load_tile`.""" + cache_key = f"tile:{layer.layer_id}-{x}-{y}-{zoom}" + + if use_cache and self.redis is not None: + cached = self.redis.get(cache_key) + if cached is not None: + LOGGER.debug("Cache hit for %s/z:%s/x:%s/y:%s", layer.layer_id, zoom, x, y) + return cached + + url = ( + layer.url.unicode_string() + .replace(quote("{x}"), str(x)) + .replace(quote("{y}"), str(y)) + .replace(quote("{z}"), str(zoom)) + ) + + # Avoid doing actual requests during tests + if url.startswith("http://localhost:0"): + LOGGER.debug("Skipping tile request for testing URL") + return b"" + + response = self.load_url(url, headers=headers) + tile_data = response.content + if use_cache and self.redis is not None: + self.redis.set(cache_key, tile_data, ex=TTL) + return tile_data + def _schedule_session_close(self): with self.lock: if self.closer: @@ -332,27 +398,15 @@ def tile_proxy(request): LOGGER.debug("Aborted attempt to contact %s due to previous timeouts", provider) raise HTTPGatewayTimeout(f"Avoiding request to {provider}") - url = ( - tile_sources[provider] - .url.unicode_string() - .replace(quote("{x}"), str(x)) - .replace(quote("{y}"), str(y)) - .replace(quote("{z}"), str(z)) - ) - # Avoid doing actual requests during tests - if url.startswith("http://localhost:0"): - LOGGER.debug("Skipping tile proxy request for testing URL") - return Response(b"", content_type="image/png") - headers = { - "user-agent": f"Fietsboek-Tile-Proxy/{__VERSION__}", - } + headers = {} from_mail = request.config.email_from if from_mail: headers["from"] = from_mail loader: ITileRequester = request.registry.getUtility(ITileRequester) try: - resp = loader.load_tile(url, headers=headers) + # We already tried the cache, so bypass it here + resp = loader.load_tile(tile_sources[provider], z, x, y, headers=headers, use_cache=False) except ReadTimeout: LOGGER.debug("Proxy timeout when accessing %r", url) request.redis.incr(timeout_tracker) @@ -365,8 +419,8 @@ def tile_proxy(request): status_code = exc.response.status_code return Response(f"Failed to get tile from {provider}", status_code=status_code) else: - request.redis.set(cache_key, resp.content, ex=TTL) - return Response(resp.content, content_type=resp.headers.get("Content-type", content_type)) + request.redis.set(cache_key, resp, ex=TTL) + return Response(resp, content_type=resp.headers.get("Content-type", content_type)) def sources_for(request: Request) -> list[TileLayerConfig]: |