make overpass API configurable

5 files changed, 98 insertions, 20 deletions

diff --git a/fietsboek/__init__.py b/fietsboek/__init__.py
index 1f21c5f..3edb243 100644
--- a/fietsboek/__init__.py
+++ b/fietsboek/__init__.py
@@ -160,6 +160,7 @@ def main(global_config, **settings):
     """This function returns a Pyramid WSGI application."""
     # Avoid a circular import by not importing at the top level
     # pylint: disable=import-outside-toplevel,cyclic-import
+    from .osm import OverpassApi
     from .views.tileproxy import TileRequester
 
     # In tests this isn't passed, so guard against it
@@ -180,6 +181,10 @@ def main(global_config, **settings):
     def redis_(request):
         return redis.from_url(request.config.redis_url)
 
+    overpass_api = OverpassApi.from_config(parsed_config)
+    def overpass_api_(_request):
+        return overpass_api
+
     def config_(_request):
         return parsed_config
 
@@ -215,6 +220,7 @@ def main(global_config, **settings):
         config.add_request_method(pages, reify=True)
         config.add_request_method(redis_, name="redis", reify=True)
         config.add_request_method(config_, name="config", reify=True)
+        config.add_request_method(overpass_api_, name="overpass_api", reify=True)
 
     config.registry.registerUtility(TileRequester(redis.from_url(parsed_config.redis_url)))
 
diff --git a/fietsboek/actions.py b/fietsboek/actions.py
index e19fa60..0c00fa9 100644
--- a/fietsboek/actions.py
+++ b/fietsboek/actions.py
@@ -27,6 +27,7 @@ from .data import DataManager, TrackDataDir
 from .geo import Path
 from .models.track import TrackType, Visibility
 from .models.user import TokenType
+from .osm import OverpassApi
 from .views.tileproxy import ITileRequester, TileRequester
 
 LOGGER = logging.getLogger(__name__)
@@ -303,6 +304,7 @@ def extract_climbs(
     track: models.Track,
     tile_requester: TileRequester,
     layer: TileLayerConfig,
+    overpass_api: OverpassApi,
     *,
     find_names: bool=True,
 ):
@@ -338,7 +340,7 @@ def extract_climbs(
 
         name = None
         if find_names:
-            name = osm.find_name(subpath)
+            name = osm.find_name(overpass_api, subpath)
 
         climb = models.Climb(
             track=track,
@@ -373,7 +375,7 @@ def extract_climbs_background(request: Request, track_id: int):
         session = request.registry["dbsession_factory"]()
         track = session.get(models.Track, track_id)
         with session:
-            extract_climbs(session, track, loader, layer)
+            extract_climbs(session, track, loader, layer, request.overpass_api)
             session.commit()
         request.redis.delete(f"bg-tasks:track-{track_id}")
         LOGGER.debug("Climb extraction done")
diff --git a/fietsboek/config.py b/fietsboek/config.py
index bd2347a..6650f08 100644
--- a/fietsboek/config.py
+++ b/fietsboek/config.py
@@ -217,6 +217,19 @@ class Config(BaseModel):
     Defaults to 0, which makes it use as many threads as there are CPU cores.
     """
 
+    overpass_url: str = Field(
+        "https://maps.mail.ru/osm/tools/overpass/api/interpreter",
+        alias="overpass.url",
+    )
+    """URL of the overpass instance to use (including the /interpreter suffix).
+    """
+
+    overpass_basic_username: str = Field("", alias="overpass.basic_auth.username")
+    """Username (if needed) for overpass authentication."""
+
+    overpass_basic_password: SecretStr = Field(SecretStr(""), alias="overpass.basic_auth.password")
+    """Password (if needed) for overpass authentication."""
+
     @field_validator("session_key")
     @classmethod
     def _good_session_key(cls, value):
diff --git a/fietsboek/osm.py b/fietsboek/osm.py
index f56cbf1..de3c827 100644
--- a/fietsboek/osm.py
+++ b/fietsboek/osm.py
@@ -4,12 +4,13 @@ import logging
 from xml.etree import ElementTree
 
 import requests
+import requests.auth
 
 from . import __VERSION__
+from .config import Config
 from .geo import Path, Point
 
 LOGGER = logging.getLogger(__name__)
-OVERPASS_API_URL = "https://maps.mail.ru/osm/tools/overpass/api/interpreter"
 HEADERS = {
     "user-agent": f"Fietsboek/{__VERSION__}",
     "accept-encoding": "gzip;q=1.0, identity;q=0.5",
@@ -22,15 +23,75 @@ MAX_DISTANCE_CITY = 750
 MAX_DISTANCE_WATER = 750
 
 
-def find_name(path: Path) -> str | None:
+class ApiError(Exception):
+    pass
+
+
+class ApiTimeout(ApiError):
+    pass
+
+
+class ApiNotOk(ApiError):
+    pass
+
+
+class OverpassApi:
+    """Helper class to access the overpass API.
+
+    This does not really provide a nice interface to use, rather, it takes care
+    of the low-level HTTP fuzz.
+    """
+
+    def __init__(self, interpreter_url: str):
+        self.session = requests.Session()
+        self.session.headers.update(HEADERS)
+        self.session.timeout = API_TIMEOUT
+        self.url = interpreter_url
+
+    def basic_auth(self, username: str, password: str):
+        """Sets the HTTP basic auth for this API.
+
+        :param username: The API username.
+        :param password: The API password.
+        """
+        self.session.auth = requests.auth.HTTPBasicAuth(username, password)
+
+    @classmethod
+    def from_config(cls, config: Config) -> "OverpassApi":
+        api = cls(config.overpass_url)
+        if config.overpass_basic_username and config.overpass_basic_password:
+            api.basic_auth(config.overpass_basic_username, config.overpass_basic_password)
+        return api
+
+    def run_script(self, script: str) -> requests.Response:
+        """Posts a overpass script to the API and returns the response.
+
+        Raises an error if the response is not 200.
+
+        :raises ApiTimeout: If the response takes too long.
+        :raises ApiNotOk: If the response code is not 200.
+        :param script: The overpass script.
+        :return: The response.
+        """
+        try:
+            with self.session.post(self.url, data={"data": script}) as response:
+                if response.status_code != 200:
+                    raise ApiNotOk(
+                        f"Overpass error response: {response.status_code} {response.reason}"
+                    )
+                return response
+        except (requests.ReadTimeout, requests.ConnectTimeout) as exc:
+            raise ApiTimeout(f"Timeout when trying to access overpass at {self.url}")
+
+
+def find_name(api: OverpassApi, path: Path) -> str | None:
     """Attempt to find a name for the given path.
 
+    :param api: The overpass API to use.
     :param path: Path to find a name for.
     :return: Name for the path.
     """
     LOGGER.debug("Starting naming process")
-    session = requests.Session()
-    session.headers.update(HEADERS)
 
     (min_lat, min_lon), (max_lat, max_lon) = path.bounds()
 
@@ -49,18 +110,12 @@ def find_name(path: Path) -> str | None:
         out;
     """
     try:
-        with session.post(
-            OVERPASS_API_URL, data={"data": overpass_script}, timeout=API_TIMEOUT
-        ) as response:
-            if response.status_code != 200:
-                LOGGER.info(
-                    "Error response from overpass: %s %s", response.status_code, response.reason
-                )
-                return None
-            data = response.content
-    except requests.ReadTimeout, requests.ConnectTimeout:
-        LOGGER.info("Overpass API timeout, skipping")
+        response = api.run_script(overpass_script)
+    except ApiError as exc:
+        LOGGER.info("%s", exc)
         return None
+    else:
+        data = response.content
 
     LOGGER.debug("Got %s bytes of OSM-XML data, parsing...", len(data))
     tree = ElementTree.fromstring(data)
diff --git a/fietsboek/scripts/fietscron.py b/fietsboek/scripts/fietscron.py
index 081b3b2..cbed349 100644
--- a/fietsboek/scripts/fietscron.py
+++ b/fietsboek/scripts/fietscron.py
@@ -17,6 +17,7 @@ from .. import hittekaart, models, osm, trackmap
 from ..config import Config
 from ..data import DataManager
 from ..models.user import TOKEN_LIFETIME
+from ..osm import OverpassApi
 from ..views.tileproxy import TileRequester
 from . import config_option
 
@@ -51,13 +52,14 @@ def cli(config):
 
     engine = create_engine(config.sqlalchemy_url)
     redis = mod_redis.from_url(config.redis_url)
+    overpass_api = OverpassApi.from_config(config)
 
     LOGGER.debug("Starting maintenance tasks")
     remove_old_uploads(engine)
     remove_old_tokens(engine)
     rebuild_cache(engine)
     build_previews(engine, data_manager, redis, config)
-    name_climbs(engine)
+    name_climbs(engine, overpass_api)
 
     redis = mod_redis.from_url(config.redis_url)
     if config.hittekaart_autogenerate:
@@ -197,7 +199,7 @@ def refill_queue(session: Session, redis: Redis):
         redis.sadd("hittekaart:queue:low", user.id)
 
 
-def name_climbs(engine: Engine):
+def name_climbs(engine: Engine, overpass_api: OverpassApi):
     """Attempts to find names for climbs that have no name assigned yet."""
     session = Session(engine)
     LOGGER.debug("Naming climbs")
@@ -206,7 +208,7 @@ def name_climbs(engine: Engine):
     for climb in session.execute(query).scalars():
         LOGGER.debug("Attempting climb %s", climb.id)
         path = climb.path()
-        name = osm.find_name(path)
+        name = osm.find_name(overpass_api, path)
         if name is not None:
             climb.automatic_name = name
             session.add(climb)