remember timeouts for tile servers

1. This makes Fietsboek faster because we don't have to wait for the
   timeout on every single request.
2. This is better for the servers, as we don't add more requests when
   they're already overloaded.

1 files changed, 20 insertions, 1 deletions

diff --git a/fietsboek/views/tileproxy.py b/fietsboek/views/tileproxy.py
index 01dc9a9..5195c91 100644
--- a/fietsboek/views/tileproxy.py
+++ b/fietsboek/views/tileproxy.py
@@ -39,7 +39,16 @@ TILE_SOURCES = {
 }
 
 TTL = datetime.timedelta(days=7)
+"""Time to live of cached tiles."""
+
 TIMEOUT = datetime.timedelta(seconds=1.5)
+"""Timeout when requesting new tiles from a source server."""
+
+PUNISHMENT_TTL = datetime.timedelta(minutes=10)
+"""Block-out period after too many requests of a server have timed out."""
+
+PUNISHMENT_THRESHOLD = 10
+"""Block a provider after that many requests have timed out."""
 
 
 @view_config(route_name='tile-proxy', http_cache=3600)
@@ -57,13 +66,21 @@ def tile_proxy(request):
 
     x, y, z = (int(request.matchdict['x']), int(request.matchdict['y']),
                int(request.matchdict['z']))
-    cache_key = f"tile-{provider}-{x}-{y}-{z}"
+    cache_key = f"tile:{provider}-{x}-{y}-{z}"
     content_type = "image/png"
 
     cached = request.redis.get(cache_key)
     if cached is not None:
         return Response(cached, content_type=content_type)
 
+    timeout_tracker = f"provider-timeout:{provider}"
+    previous_timeouts = int(request.redis.get(timeout_tracker) or "0")
+    if previous_timeouts > PUNISHMENT_THRESHOLD:
+        # We've gotten too many timeouts from this provider recently, so avoid
+        # contacting it in the first place.
+        LOGGER.debug("Aborted attempt to contact %s due to previous timeouts", provider)
+        return HTTPGatewayTimeout(f"Avoiding request to {provider}")
+
     url = TILE_SOURCES[provider].format(x=x, y=y, z=z, s=random.choice("abc"))
     headers = {
         "user-agent": f"Fietsboek-Tile-Proxy/{__VERSION__}",
@@ -76,6 +93,8 @@ def tile_proxy(request):
         resp = requests.get(url, headers=headers, timeout=TIMEOUT.total_seconds())
     except ReadTimeout:
         LOGGER.debug("Proxy timeout when accessing %r", url)
+        request.redis.incr(timeout_tracker)
+        request.redis.expire(timeout_tracker, PUNISHMENT_TTL)
         return HTTPGatewayTimeout(f"No response in time from {url}")
     else:
         request.redis.set(cache_key, resp.content, ex=TTL)